Merge pull request #135 from nautobot/feature_delices_merge_from_develop

Feature delices merge from develop

.flake8

@@ -1,7 +1,9 @@
 [flake8]
+# E501 - Line length is enforced by Black, so flake8 doesn't need to check it
+# W503 - Black disagrees with this rule, as does PEP 8; Black wins
 ignore =
-  E501,  # Line length is enforced by Black, so flake8 doesn't need to check it
-  W503   # Black disagrees with this rule, as does PEP 8; Black wins
+  E501,
+  W503
 exclude = 
   migrations,
   __pycache__,

.github/workflows/ci.yml

@@ -1,6 +1,6 @@
 ---
 name: "CI"
-concurrency: # Cancel any existing runs of this workflow for this same PR
+concurrency:  # Cancel any existing runs of this workflow for this same PR
   group: "${{ github.workflow }}-${{ github.ref }}"
   cancel-in-progress: true
 on: # yamllint disable-line rule:truthy rule:comments

development/development.env

@@ -39,3 +39,6 @@ MYSQL_DATABASE=${NAUTOBOT_DB_NAME}
 MYSQL_ROOT_HOST=%
 
 DESIGN_BUILDER_ENABLE_BGP=True
+
+NAUTOBOT_CELERY_TASK_SOFT_TIME_LIMIT=86400
+NAUTOBOT_CELERY_TASK_TIME_LIMIT=86401

development/docker-compose.test-designs.yml

@@ -0,0 +1,9 @@
+---
+version: "3.8"
+services:
+  nautobot:
+    volumes:
+      - "../nautobot_design_builder/tests/designs:/opt/nautobot/jobs"
+  worker:
+    volumes:
+      - "../nautobot_design_builder/tests/designs:/opt/nautobot/jobs"

docs/admin/release_notes/version_1.1.md

@@ -0,0 +1,23 @@
+# v1.1.0 Release Notes
+
+## Release Overview
+
+The 1.1 release of Design Builder mostly includes performance improvements in the implementation system. There is a breaking change related to one-to-one relationships. Prior to the 1.1 release, one-to-one relationships were not saved until after the parent object was saved. The performance optimization work revealed this as a performance issue and now one-to-one relationships are treated as simple foreign keys. Since foreign key saves are not deferred by default, it may now be necessary to explicitly specify deferring the save operation. A new `deferred` attribute has been introduced that causes design builder to defer saving the foreign-key relationship until after the parent object has been saved. The one known case that is affected by this change is when setting a device primary IP when the IP itself is created as a member of an interface in the same device block. See unit tests and design examples for further explanation.
+
+Additionally, the `design.Builder` class has been renamed to `design.Environment` to better reflect what that class does. A `Builder` alias has been added to `Environment` for backwards compatibility with a deprecation warning.
+
+## [v1.1.0] - 2024-04
+
+### Added
+
+- Added `deferred` attribute to allow deferral of field assignment. See notes in the Changed section.
+
+- Added `model_metadata` attribute to models. At the moment, this provides the ability to specify additional arguments passed to the `save` method of models being updated. The known use case for this is in creating Git repositories in Nautobot 1.x where `trigger_resync` must be `False`. In the future, additional fields will be added to `model_metadata` to provide new functionality.
+
+### Changed
+
+- Renamed `nautobot_design_builder.design.Builder` to `nautobot_design_builder.Environment` - aliased original name with deprecation warning.
+
+- Any designs that set `OneToOne` relationships (such as device `primary_ip4`) may now need a `deferred: true` statement in their design for those fields. Previously, `OneToOne` relationships were always deferred and this is usually unnecessary. Any deferrals must now be explicit.
+
+- Design reports are now saved to the file `report.md` for Nautobot 2.x installations.

docs/dev/code_reference/fields.md

@@ -0,0 +1 @@
+::: nautobot_design_builder.fields

examples/custom_design/designs/l3vpn/context/__init__.py

@@ -1,9 +1,8 @@
 from django.core.exceptions import ObjectDoesNotExist
 import ipaddress
-from functools import lru_cache
 
-from nautobot.dcim.models import Device, Interface
-from nautobot.ipam.models import VRF, Prefix
+from nautobot.dcim.models import Device
+from nautobot.ipam.models import VRF
 
 from nautobot_design_builder.context import Context, context_file
 
@@ -19,20 +18,6 @@ class L3VPNContext(Context):
     def __hash__(self):
         return hash((self.pe.name, self.ce.name, self.customer_name))
 
-    @lru_cache
-    def get_l3vpn_prefix(self, parent_prefix, prefix_length):
-        tag = self.design_instance_tag
-        if tag:
-            existing_prefix = Prefix.objects.filter(tags__in=[tag], prefix_length=30).first()
-            if existing_prefix:
-                return str(existing_prefix)
-
-        for new_prefix in ipaddress.ip_network(parent_prefix).subnets(new_prefix=prefix_length):
-            try:
-                Prefix.objects.get(prefix=str(new_prefix))
-            except ObjectDoesNotExist:
-                return new_prefix
-
     def get_customer_id(self, customer_name, l3vpn_asn):
         try:
             vrf = VRF.objects.get(description=f"VRF for customer {customer_name}")
@@ -44,16 +29,6 @@ def get_customer_id(self, customer_name, l3vpn_asn):
             new_id = int(last_vrf.name.split(":")[-1]) + 1
             return str(new_id)
 
-    def get_interface_name(self, device):
-        root_interface_name = "GigabitEthernet"
-        interfaces = Interface.objects.filter(name__contains=root_interface_name, device=device)
-        tag = self.design_instance_tag
-        if tag:
-            existing_interface = interfaces.filter(tags__in=[tag]).first()
-            if existing_interface:
-                return existing_interface.name
-        return f"{root_interface_name}1/{len(interfaces) + 1}"
-
     def get_ip_address(self, prefix, offset):
         net_prefix = ipaddress.ip_network(prefix)
         for count, host in enumerate(net_prefix):

examples/custom_design/designs/l3vpn/designs/0001_ipam.yaml.j2

@@ -1,14 +1,25 @@
 ---
 
-vrfs:
-  - "!create_or_update:name": "{{ l3vpn_asn }}:{{ get_customer_id(customer_name, l3vpn_asn) }}"
-    description: "VRF for customer {{ customer_name }}"
-    "!ref": "my_vrf"
-
+tags:
+    - "!create_or_update:name": "VRF Prefix"
+      "slug": "vrf_prefix"
+    - "!create_or_update:name": "VRF Interface"
+      "slug": "vrf_interface"
 
 prefixes:
     - "!create_or_update:prefix": "{{ l3vpn_prefix }}"
       status__name: "Reserved"
-    - "!create_or_update:prefix": "{{ get_l3vpn_prefix(l3vpn_prefix, l3vpn_prefix_length) }}"
-      status__name: "Reserved"
-      vrf: "!ref:my_vrf"
+
+vrfs:
+  - "!create_or_update:name": "{{ l3vpn_asn }}:{{ get_customer_id(customer_name, l3vpn_asn) }}"
+    description: "VRF for customer {{ customer_name }}"
+    prefixes:
+      - "!next_prefix":
+          identified_by:
+            tags__name: "VRF Prefix"
+          prefix: "{{ l3vpn_prefix }}"
+          length: 30
+        status__name: "Reserved"
+        tags:
+          - {"!get:name": "VRF Prefix"}
+        "!ref": "l3vpn_p2p_prefix"

examples/custom_design/designs/l3vpn/designs/0002_devices.yaml.j2

@@ -8,18 +8,22 @@
       "mpls_router": true,
     }
     interfaces:
-      - "!create_or_update:name": "{{ get_interface_name(device) }}"
+      - "!next_interface": {}
         status__name: "Planned"
         type: "other"
         {% if offset == 2 %}
         "!connect_cable":
           status__name: "Planned"
-          to:
-            device__name: "{{ other_device.name }}"
-            name: "{{ get_interface_name(other_device) }}"
+          to: "!ref:other_interface"
+        {% else %}
+        "!ref": "other_interface"
         {% endif %}
+        tags:
+          - {"!get:name": "VRF Interface"}
         ip_addresses:
-        - "!create_or_update:address": "{{ get_ip_address(get_l3vpn_prefix(l3vpn_prefix, l3vpn_prefix_length), offset) }}"
+        - "!child_prefix:address":
+            parent: "!ref:l3vpn_p2p_prefix"
+            offset: "0.0.0.{{ offset }}/30"
           status__name: "Reserved"
 
 {% endmacro %}

examples/custom_design/designs/l3vpn/jobs.py

@@ -2,15 +2,50 @@
 
 from django.core.exceptions import ValidationError
 
-from nautobot.dcim.models import Device
+from nautobot.dcim.models import Device, Interface
 from nautobot.extras.jobs import ObjectVar, StringVar
 
 from nautobot_design_builder.design_job import DesignJob
+from nautobot_design_builder.design import ModelInstance
+from nautobot_design_builder.ext import AttributeExtension
 from nautobot_design_builder.contrib import ext
 
 from .context import L3VPNContext
 
 
+class NextInterfaceExtension(AttributeExtension):
+    """Attribute extension to calculate the next available interface name."""
+
+    tag = "next_interface"
+
+    def attribute(self, *args, value, model_instance: ModelInstance) -> dict:
+        """Determine the next available interface name.
+
+        Args:
+            *args: Any additional arguments following the tag name. These are `:` delimited.
+            value (Any): The value of the data structure at this key's point in the design YAML. This could be a scalar, a dict or a list.
+            model_instance (ModelInstance): Object is the ModelInstance that would ultimately contain the values.
+
+        Returns:
+            dict: Dictionary with the new interface name `{"!create_or_update:name": new_interface_name}
+        """
+        root_interface_name = "GigabitEthernet"
+        previous_interfaces = self.environment.design_instance.get_design_objects(Interface).values_list(
+            "id", flat=True
+        )
+        interfaces = model_instance.relationship_manager.filter(
+            name__startswith="GigabitEthernet",
+        )
+        existing_interface = interfaces.filter(
+            pk__in=previous_interfaces,
+            tags__name="VRF Interface",
+        ).first()
+        if existing_interface:
+            model_instance.instance = existing_interface
+            return {"!create_or_update:name": existing_interface.name}
+        return {"!create_or_update:name": f"{root_interface_name}1/{len(interfaces) + 1}"}
+
+
 class L3vpnDesign(DesignJob):
     """Create a l3vpn connection."""
 
@@ -38,7 +73,12 @@ class Meta:
             "designs/0002_devices.yaml.j2",
         ]
         context_class = L3VPNContext
-        extensions = [ext.CableConnectionExtension]
+        extensions = [
+            ext.CableConnectionExtension,
+            ext.NextPrefixExtension,
+            NextInterfaceExtension,
+            ext.ChildPrefixExtension,
+        ]
 
     @staticmethod
     def validate_data_logic(data):

mkdocs.yml

@@ -118,6 +118,7 @@ nav:
       - Release Notes:
           - "admin/release_notes/index.md"
           - v1.0: "admin/release_notes/version_1.0.md"
+          - v1.1: "admin/release_notes/version_1.1.md"
   - Developer Guide:
       - Extending the App: "dev/extending.md"
       - Contributing to the App: "dev/contributing.md"
@@ -127,6 +128,7 @@ nav:
           - Package: "dev/code_reference/package.md"
           - Context: "dev/code_reference/context.md"
           - Design Builder: "dev/code_reference/design.md"
+          - Field Descriptors: "dev/code_reference/fields.md"
           - Design Job: "dev/code_reference/design_job.md"
           - Jinja Rendering: "dev/code_reference/jinja2.md"
           - Template Extensions: "dev/code_reference/ext.md"

nautobot_design_builder/context.py

@@ -3,13 +3,12 @@
 from functools import cached_property
 from collections import UserList, UserDict, UserString
 import inspect
-from typing import Any, Union
+from typing import Any
 import yaml
 
 from jinja2.nativetypes import NativeEnvironment
 
 from nautobot.extras.models import JobResult
-from nautobot.extras.models import Tag
 
 from nautobot_design_builder.errors import DesignValidationError
 from nautobot_design_builder.jinja2 import new_template_environment
@@ -371,14 +370,6 @@ def validate(self):
         if len(errors) > 0:
             raise DesignValidationError("\n".join(errors))
 
-    @property
-    def design_instance_tag(self) -> Union[Tag, None]:
-        """Returns the `Tag` of the design instance if exists."""
-        try:
-            return Tag.objects.get(name__contains=self._instance_name)
-        except Tag.DoesNotExist:
-            return None
-
     @property
     def _instance_name(self):
         if nautobot_version < "2.0.0":