Refactor(eos_designs): Move eos_designs Python modules to PyAVD (#4120)

.pre-commit-config.yaml

@@ -105,12 +105,21 @@ repos:
     rev: "v3.1.1"
     hooks:
       - id: pylint # Use pylintrc file in repository
-        name: Check for Linting errors on Python files with settings defined in `pylintrc`.
+        name: Check for Linting errors on Python files outside the Ansible collection.
         description: This hook runs pylint.
         types: [python]
         args:
           # Suppress duplicate code for modules header
           - -d duplicate-code
+      - id: pylint # Use pylintrc file in Ansible collection
+        name: Check for Linting errors on Python files in the Ansible collection.
+        description: This hook runs pylint.
+        types: [python]
+        files: ansible_collections/arista/avd/
+        args:
+          - --rcfile=ansible_collections/arista/avd/pylintrc
+          # Suppress duplicate code for modules header
+          - -d duplicate-code
 
   - repo: https://github.com/adrienverge/yamllint.git
     rev: "v1.35.1"

ansible_collections/arista/avd/docs/contribution/eos_designs_internal_notes.md

@@ -230,18 +230,7 @@ which at the time where `eos_designs_structured_config` run, is a nested `dict`.
 
 Subclasses are typically using Mixin classes to split all the attributes/`cached_properties` into manageable files.
 
-Source code:
-
-- [AvdStructuredConfigBase](../../roles/eos_designs/python_modules/base/__init__.py)
-  (Unfortunate naming. Base here refers to base configurations. Not a Base class.)
-- [AvdStructuredConfigConnectedEndpoints](../../roles/eos_designs/python_modules/connected_endpoints/__init__.py)
-- [AvdStructuredConfigCoreInterfacesAndL3Edge](../../roles/eos_designs/python_modules/core_interfaces_and_l3_edge/__init__.py)
-- [AvdStructuredConfigCustomStructuredConfiguration](../../roles/eos_designs/python_modules/custom_structured_configuration/__init__.py)
-- [AvdStructuredConfigInbandManagement](../../roles/eos_designs/python_modules/inband_management/__init__.py)
-- [AvdStructuredConfigMlag](../../roles/eos_designs/python_modules/mlag/__init__.py)
-- [AvdStructuredConfigNetworkServices](../../roles/eos_designs/python_modules/network_services/__init__.py)
-- [AvdStructuredConfigOverlay](../../roles/eos_designs/python_modules/overlay/__init__.py)
-- [AvdStructuredConfigUnderlay](../../roles/eos_designs/python_modules/underlay/__init__.py)
+See the source code [here](https://github.com/aristanetworks/avd/tree/devel/python-avd/pyavd/_eos_designs/structured_config)
 
 ```mermaid
 classDiagram

ansible_collections/arista/avd/docs/plugins/Modules_and_action_plugins/yaml_templates_to_facts.md

@@ -51,7 +51,7 @@ Set facts from YAML produced by Jinja2 templates
   arista.avd.yaml_templates_to_facts:
     root_key: structured_config
     templates:
-      - python_module: "ansible_collections.arista.avd.roles.eos_designs.python_modules.base"
+      - python_module: "pyavd._eos_designs.structured_config.base"
         python_class_name: "AvdStructuredConfig"
       - template: "mlag/main.j2"
       - template: "designs/underlay/main.j2"

ansible_collections/arista/avd/molecule/eos_designs_negative_unit_tests/inventory/host_vars/missing-mlag-peer.yml

@@ -12,4 +12,7 @@ l2leaf:
         - name: some-missing-device
           id: 2
 
-expected_error_message: "Facts not found for node 'some-missing-device'. Something in the input vars is pointing to this node. Check that 'some-missing-device' is in the inventory and is part of the group set by 'fabric_name'. Node is required but was not found for host 'missing-mlag-peer'"
+expected_error_message: >-
+  Facts not found for node 'some-missing-device'. Something in the input vars is pointing to this node.
+  Check that 'some-missing-device' is in the inventory and is part of the group set by 'fabric_name'.
+  Node is required but was not found for host 'missing-mlag-peer'

ansible_collections/arista/avd/molecule/eos_designs_unit_tests/custom_modules/custom_interface_descriptions.py

@@ -4,6 +4,8 @@
 from functools import cached_property
 
 from ansible_collections.arista.avd.plugins.plugin_utils.utils import get
+
+# TODO: AVD5.0 change this to import from PyAVD instead and remove the python_modules folder.
 from ansible_collections.arista.avd.roles.eos_designs.python_modules.interface_descriptions import AvdInterfaceDescriptions
 
 

ansible_collections/arista/avd/molecule/eos_designs_unit_tests/custom_modules/custom_interface_descriptions_with_data.py

@@ -4,6 +4,8 @@
 from functools import cached_property
 
 from ansible_collections.arista.avd.plugins.plugin_utils.utils import get
+
+# TODO: AVD5.0 change this to import from PyAVD instead and remove the python_modules folder.
 from ansible_collections.arista.avd.roles.eos_designs.python_modules.interface_descriptions import AvdInterfaceDescriptions, InterfaceDescriptionData
 
 

ansible_collections/arista/avd/molecule/eos_designs_unit_tests/custom_modules/custom_ip_addressing.py

@@ -3,6 +3,7 @@
 # that can be found in the LICENSE file.
 from functools import cached_property
 
+# TODO: AVD5.0 change this to import from PyAVD instead and remove the python_modules folder.
 from ansible_collections.arista.avd.roles.eos_designs.python_modules.ip_addressing import AvdIpAddressing
 
 

ansible_collections/arista/avd/molecule/eos_designs_unit_tests/intended/configs/UNDERLAY-MULTICAST-L3LEAF2B.cfg

@@ -191,7 +191,7 @@ router pim sparse-mode
 router msdp
    originator-id local-interface Loopback0
    !
-   peer 192.168.255.6
+   peer 192.168.255.5
       mesh-group ANYCAST-RP
       local-interface Loopback0
       description UNDERLAY-MULTICAST-L3LEAF2A

ansible_collections/arista/avd/molecule/eos_designs_unit_tests/intended/structured_configs/UNDERLAY-MULTICAST-L3LEAF2B.yml

@@ -220,7 +220,7 @@ prefix_lists:
 router_msdp:
   originator_id_local_interface: Loopback0
   peers:
-  - ipv4_address: 192.168.255.6
+  - ipv4_address: 192.168.255.5
     local_interface: Loopback0
     description: UNDERLAY-MULTICAST-L3LEAF2A
     mesh_groups:

ansible_collections/arista/avd/plugins/action/eos_designs_facts.py

@@ -3,8 +3,6 @@
 # that can be found in the LICENSE file.
 from __future__ import absolute_import, division, print_function
 
-from ansible_collections.arista.avd.plugins.plugin_utils.pyavd_wrappers import RaiseOnUse
-
 __metaclass__ = type
 
 import cProfile
@@ -15,13 +13,15 @@
 
 from ansible_collections.arista.avd.plugins.plugin_utils.eos_designs_shared_utils import SharedUtils
 from ansible_collections.arista.avd.plugins.plugin_utils.errors import AristaAvdMissingVariableError
+from ansible_collections.arista.avd.plugins.plugin_utils.pyavd_wrappers import RaiseOnUse
 from ansible_collections.arista.avd.plugins.plugin_utils.schema.avdschematools import AvdSchemaTools
 from ansible_collections.arista.avd.plugins.plugin_utils.utils import get_templar
 
 PLUGIN_NAME = "arista.avd.eos_designs_facts"
 
 try:
     from pyavd._eos_designs.eos_designs_facts import EosDesignsFacts
+    from pyavd.vendor.errors.errors import AristaAvdMissingVariableError as PyavdMissingVariableError
 except ImportError as e:
     EosDesignsFacts = RaiseOnUse(
         AnsibleActionFail(
@@ -204,7 +204,7 @@ def render_avd_switch_facts(self, avd_switch_facts_instances: dict):
         for host in avd_switch_facts_instances:
             try:
                 rendered_facts[host] = {"switch": avd_switch_facts_instances[host]["switch"].render()}
-            except AristaAvdMissingVariableError as e:
+            except (AristaAvdMissingVariableError, PyavdMissingVariableError) as e:
                 raise AnsibleActionFail(f"{e} is required but was not found for host '{host}'") from e
 
             # If the argument 'template_output' is set, run the output data through jinja2 rendering.

ansible_collections/arista/avd/plugins/action/eos_designs_structured_config.py

@@ -15,11 +15,22 @@
 from ansible.plugins.action import ActionBase, display
 
 from ansible_collections.arista.avd.plugins.plugin_utils.merge import merge
+from ansible_collections.arista.avd.plugins.plugin_utils.pyavd_wrappers import RaiseOnUse
 from ansible_collections.arista.avd.plugins.plugin_utils.schema.avdschematools import AvdSchemaTools
 from ansible_collections.arista.avd.plugins.plugin_utils.strip_empties import strip_null_from_data
 from ansible_collections.arista.avd.plugins.plugin_utils.utils import get, get_templar
 from ansible_collections.arista.avd.plugins.plugin_utils.utils import template as templater
-from ansible_collections.arista.avd.roles.eos_designs.python_modules.get_structured_config import get_structured_config
+
+PLUGIN_NAME = "arista.avd.eos_designs_structured_config"
+try:
+    from pyavd._eos_designs.structured_config import get_structured_config
+except ImportError as e:
+    get_structured_config = RaiseOnUse(
+        AnsibleActionFail(
+            f"The '{PLUGIN_NAME}' plugin requires the 'pyavd' Python library. Got import error",
+            orig_exc=e,
+        )
+    )
 
 
 class ActionModule(ActionBase):
@@ -79,7 +90,7 @@ def run(self, tmp=None, task_vars=None):
             plugin_name="arista.avd.eos_cli_config_gen",
         )
 
-        # Get Structured Config from builtin eos_designs python_modules
+        # Get Structured Config from modules in PyAVD using internal api so we can supply our own templar
         try:
             output = get_structured_config(
                 vars=dict(task_vars),
@@ -101,7 +112,7 @@ def run(self, tmp=None, task_vars=None):
         # Any var assignments will end up in output, so all other objects are protected.
         template_vars = ChainMap(output, task_vars)
 
-        # eos_designs_custom_templates can contain a list of jinja templates to run after the builtin eos_designs python_modules
+        # eos_designs_custom_templates can contain a list of jinja templates to run after PyAVD
         for template_item in eos_designs_custom_templates:
             template_options = template_item.get("options", {})
             list_merge = template_options.get("list_merge", "append_rp")

ansible_collections/arista/avd/plugins/action/yaml_templates_to_facts.py

@@ -147,7 +147,7 @@ def run(self, tmp=None, task_vars=None):
         if debug:
             avd_yaml_templates_to_facts_debug = task_vars.get("avd_yaml_templates_to_facts_debug", [])
 
-        # template_list can contain templates or python_modules
+        # template_list can contain templates or python modules
         for template_item in template_list:
             if debug:
                 debug_item = template_item

ansible_collections/arista/avd/plugins/modules/yaml_templates_to_facts.py

@@ -131,7 +131,7 @@
   arista.avd.yaml_templates_to_facts:
     root_key: structured_config
     templates:
-      - python_module: "ansible_collections.arista.avd.roles.eos_designs.python_modules.base"
+      - python_module: "pyavd._eos_designs.structured_config.base"
         python_class_name: "AvdStructuredConfig"
       - template: "mlag/main.j2"
       - template: "designs/underlay/main.j2"

ansible_collections/arista/avd/plugins/plugin_utils/eos_designs_facts/MOVED_TO_PYAVD.txt

@@ -0,0 +1 @@
+The eos_designs_facts Python module has been moved to PyAVD in the folder /python-avd/pyavd/_eos_designs/eos_designs_facts/

ansible_collections/arista/avd/plugins/plugin_utils/eos_designs_shared_utils/shared_utils.py

@@ -52,7 +52,7 @@ class SharedUtils(
     FlowTrackingMixin,
 ):
     """
-    Class with commonly used methods / cached_properties to be shared between all the python_modules
+    Class with commonly used methods / cached_properties to be shared between all the python modules
     loaded in eos_designs.
 
     This class is instantiated in 'EosDesignsFacts' class and set as 'shared_utils' property.

ansible_collections/arista/avd/plugins/plugin_utils/utils/template.py

@@ -1,9 +1,6 @@
 # Copyright (c) 2023-2024 Arista Networks, Inc.
 # Use of this source code is governed by the Apache License 2.0
 # that can be found in the LICENSE file.
-from ansible.module_utils._text import to_text
-
-
 def template(template_file, template_vars, templar):
     """
     Run Ansible Templar with template file.
@@ -30,6 +27,14 @@ def template(template_file, template_vars, templar):
     str
         The rendered template
     """
+    if templar is None:
+        raise NotImplementedError("Jinja Templating is not implemented in pyavd")
+
+    # We only get here when running from Ansible, so it is safe to import from ansible.
+    # pylint: disable=import-outside-toplevel
+    from ansible.module_utils._text import to_text
+
+    # pylint: enable=import-outside-toplevel
 
     dataloader = templar._loader
     searchpath = templar.environment.loader.searchpath

ansible_collections/arista/avd/pylintrc

@@ -0,0 +1,131 @@
+# Configuration applied to collection by ansible-test
+# https://github.com/ansible/ansible/blob/devel/test/lib/ansible_test/_data/sanity/pylint/config/collection.cfg
+
+[MASTER]
+ignore-paths=
+  # The cv_client api is generated from proto files, so it should not be linted.
+  plugins/plugin_utils/cv_client/api/
+
+[MESSAGES CONTROL]
+disable=
+    abstract-method,
+    access-member-before-definition,
+    arguments-differ,
+    assignment-from-no-return,
+    assignment-from-none,
+    attribute-defined-outside-init,
+    bad-indentation,
+    bad-mcs-classmethod-argument,
+    broad-except,
+    c-extension-no-member,
+    cell-var-from-loop,
+    chained-comparison,
+    comparison-with-callable,
+    consider-iterating-dictionary,
+    consider-merging-isinstance,
+    consider-using-dict-comprehension,
+    consider-using-enumerate,
+    consider-using-get,
+    consider-using-in,
+    consider-using-set-comprehension,
+    consider-using-ternary,
+    deprecated-method,
+    deprecated-module,
+    eval-used,
+    exec-used,
+    expression-not-assigned,
+    fixme,
+    function-redefined,
+    global-statement,
+    global-variable-undefined,
+    import-error,
+    import-self,
+    inconsistent-return-statements,
+    invalid-envvar-default,
+    invalid-name,
+    invalid-sequence-index,
+    keyword-arg-before-vararg,
+    len-as-condition,
+    line-too-long,
+    literal-comparison,
+    locally-disabled,
+    method-hidden,
+    missing-docstring,
+    no-else-raise,
+    no-else-return,
+    no-member,
+    no-name-in-module,
+    no-value-for-parameter,
+    non-iterator-returned,
+    not-a-mapping,
+    not-an-iterable,
+    not-callable,
+    pointless-statement,
+    pointless-string-statement,
+    possibly-unused-variable,
+    protected-access,
+    redefined-argument-from-local,
+    redefined-builtin,
+    redefined-outer-name,
+    reimported,
+    relative-beyond-top-level,  # https://github.com/PyCQA/pylint/issues/2967
+    signature-differs,
+    simplifiable-if-expression,
+    simplifiable-if-statement,
+    subprocess-popen-preexec-fn,
+    super-init-not-called,
+    superfluous-parens,
+    too-few-public-methods,
+    too-many-ancestors,
+    too-many-arguments,
+    too-many-boolean-expressions,
+    too-many-branches,
+    too-many-function-args,
+    too-many-instance-attributes,
+    too-many-lines,
+    too-many-locals,
+    too-many-nested-blocks,
+    too-many-public-methods,
+    too-many-return-statements,
+    too-many-statements,
+    trailing-comma-tuple,
+    trailing-comma-tuple,
+    try-except-raise,
+    unbalanced-tuple-unpacking,
+    undefined-loop-variable,
+    unexpected-keyword-arg,
+    ungrouped-imports,
+    unidiomatic-typecheck,
+    unnecessary-pass,
+    unsubscriptable-object,
+    unsupported-assignment-operation,
+    unsupported-delete-operation,
+    unsupported-membership-test,
+    unused-argument,
+    unused-import,
+    unused-variable,
+    used-before-assignment,
+    useless-object-inheritance,
+    useless-return,
+    useless-super-delegation,
+    wrong-import-order,
+    wrong-import-position,
+
+[BASIC]
+bad-names=foo,
+          bar,
+          baz,
+          toto,
+          tutu,
+          tata,
+          _,
+
+good-names=i,
+           j,
+           k,
+           ex,
+           Run,
+
+[TYPECHECK]
+ignored-modules=
+    _MovedItems,

ansible_collections/arista/avd/roles/eos_designs/python_modules/MOVED_TO_PYAVD.txt

@@ -0,0 +1 @@
+All Python modules have been moved to PyAVD in the folder /python-avd/pyavd/_eos_designs/structured_config/