[ltm-1.6] Import mode for existing data (Leo)

docs/user/design_lifecycle.md

@@ -83,6 +83,30 @@ The decommissioning feature takes into account potential dependencies between de
 
 Once a design deployment is decommissioned, it's still visible in the API/UI to check the history of changes but without any active relationship with Nautobot objects (in a "Decommissioned" status). Once decommissioned, the design deployment can be deleted completely from Nautobot.
 
+There is a decommissioning mode to only remove the link between the design objects and the design deployment without actually reverting the state of the objects. Decommissioning, with the `delete` checkbox _not_ set, is only removing the references but keeping the data.
+
 The decommissioning job outputs a log with all the detailed operation reverting to previous state (i.e., deleting or recovering original data):
 
 ![design-deployment-decommissioning](../images/screenshots/design-deployment-decommissioning.png)
+
+### Design Deployment Import
+
+Design Builder addresses
+
+- greenfield use cases by creating new data from a design
+- brownfield use cases by importing existing data related to a new design deployment
+
+In the "deployment" mode, a design deployment tracks all the objects and attributes that are "owned" by it. With the import functionality, orphan objects and attributes will be incorporated into a new design deployment as if they have been set by it.
+
+The import logic works like this:
+
+1. If the object that we reference doesn't exist, normal design creation logic applies
+2. If an object that we want to "create" already exists, normal design creation logic _also_ applies
+3. If an object that we want to "create_or_update" already exists
+    - If it's not owned by another design deployment, we get "full_control" of it and of all the attributes that we define (including the identifiers)
+    - If it already has an owner, we don't claim ownership of the object, but we still may claim the attributes, except the identifiers
+4. If an object that we want to "update" already exists
+    - There is no claim for "full_control" ownership
+    - There is a claim for the attributes, except the identifiers
+5. In all cases, the attributes that a design is trying to update are claimed. These attributes can't be claimed by any other design. If so, the import fails pointing to the conflict dependency.
+6. The imported changes (attributes) show the same old and new value because we can't infer which was the previous value (in most cases, it would be `null` but we can't be sure)

nautobot_design_builder/design.py

@@ -43,12 +43,13 @@ class Journal:
     will only be in each of those indices at most once.
     """
 
-    def __init__(self, change_set: models.ChangeSet = None):
+    def __init__(self, change_set: models.ChangeSet = None, import_mode=False):
         """Constructor for Journal object."""
         self.index = set()
         self.created = defaultdict(set)
         self.updated = defaultdict(set)
         self.change_set = change_set
+        self.import_mode = import_mode
 
     def log(self, model: "ModelInstance"):
         """Log that a model has been created or updated.
@@ -59,7 +60,7 @@ def log(self, model: "ModelInstance"):
         instance = model.instance
         model_type = instance.__class__
         if self.change_set:
-            self.change_set.log(model)
+            self.change_set.log(model, self.import_mode)
 
         if instance.pk not in self.index:
             self.index.add(instance.pk)
@@ -128,6 +129,8 @@ class ModelMetadata:  # pylint: disable=too-many-instance-attributes
     CREATE_OR_UPDATE = "create_or_update"
 
     ACTION_CHOICES = [GET, CREATE, UPDATE, CREATE_OR_UPDATE]
+    # Actions that work with import mode
+    IMPORTABLE_ACTION_CHOICES = [UPDATE, CREATE_OR_UPDATE]
 
     def __init__(self, model_instance: "ModelInstance", **kwargs):
         """Initialize the metadata object for a given model instance.
@@ -437,7 +440,6 @@ def __init__(
         self.environment = environment
         self.instance: Model = None
         self.metadata = ModelMetadata(self, **attributes.pop("model_metadata", {}))
-
         self._parent = parent
         self.refresh_custom_relationships()
         self.relationship_manager = relationship_manager
@@ -681,7 +683,11 @@ def __new__(cls, *args, **kwargs):
         return object.__new__(cls)
 
     def __init__(
-        self, job_result: JobResult = None, extensions: List[ext.Extension] = None, change_set: models.ChangeSet = None
+        self,
+        job_result: JobResult = None,
+        extensions: List[ext.Extension] = None,
+        change_set: models.ChangeSet = None,
+        import_mode=False,
     ):
         """Create a new build environment for implementing designs.
 
@@ -698,7 +704,7 @@ def __init__(
         """
         self.job_result = job_result
         self.logger = get_logger(__name__, self.job_result)
-
+        self.import_mode = import_mode
         self.extensions = {
             "extensions": [],
             "attribute": {},
@@ -722,7 +728,7 @@ def __init__(
 
             self.extensions["extensions"].append(extn)
 
-        self.journal = Journal(change_set=change_set)
+        self.journal = Journal(change_set=change_set, import_mode=import_mode)
         if change_set:
             self.deployment = change_set.deployment
 
@@ -842,6 +848,7 @@ def resolve_values(self, value: Union[list, dict, str]) -> Any:
                 value[k] = self.resolve_value(item)
         return value
 
+    # TODO(2.x): Rename to `_create_or_import-objects`
     def _create_objects(self, model_class: Type[ModelInstance], objects: Union[List[Any], Dict[str, Any]]):
         if isinstance(objects, dict):
             model = model_class(self, objects)

nautobot_design_builder/design_job.py

@@ -14,7 +14,7 @@
 from jinja2 import TemplateError
 
 from nautobot.extras.models import Status
-from nautobot.extras.jobs import Job, JobForm, StringVar
+from nautobot.extras.jobs import Job, JobForm, StringVar, BooleanVar
 
 from nautobot_design_builder.errors import DesignImplementationError, DesignModelError
 from nautobot_design_builder.jinja2 import new_template_environment
@@ -77,7 +77,7 @@ def determine_deployment_name(cls, data):
         deployment_name_field = cls.deployment_name_field()
         if deployment_name_field is None:
             if "deployment_name" not in data:
-                raise DesignImplementationError("No instance name was provided for the deployment.")
+                raise DesignImplementationError("No Deployment name was provided for the deployment.")
             return data["deployment_name"]
         return data[deployment_name_field]
 
@@ -95,6 +95,7 @@ def _get_vars(cls):
                     label="Deployment Name",
                     max_length=models.DESIGN_NAME_MAX_LENGTH,
                 )
+            cls_vars["import_mode"] = BooleanVar(label="Import Mode", default=False)
 
         cls_vars.update(super()._get_vars())
         return cls_vars
@@ -270,7 +271,6 @@ def _run_in_transaction(self, **kwargs):  # pylint: disable=too-many-branches, t
         """
         sid = transaction.savepoint()
 
-        self.log_info(message=f"Building {getattr(self.Meta, 'name')}")
         extensions = getattr(self.Meta, "extensions", [])
 
         design_files = None
@@ -282,6 +282,7 @@ def _run_in_transaction(self, **kwargs):  # pylint: disable=too-many-branches, t
 
         self.job_result.job_kwargs = {"data": self.serialize_data(data)}
 
+        data["import_mode"] = self.is_deployment_job() and data.get("import_mode", False)
         data["deployment_name"] = self.determine_deployment_name(data)
         change_set, previous_change_set = self._setup_changeset(data["deployment_name"])
         self.log_info(message=f"Building {getattr(self.Meta, 'name')}")
@@ -290,8 +291,12 @@ def _run_in_transaction(self, **kwargs):  # pylint: disable=too-many-branches, t
             job_result=self.job_result,
             extensions=extensions,
             change_set=change_set,
+            import_mode=data["import_mode"],
         )
 
+        if data["import_mode"]:
+            self.log_info(message=f'Running in import mode for {data["deployment_name"]}.')
+
         design_files = None
 
         if hasattr(self.Meta, "context_class"):

nautobot_design_builder/fields.py

@@ -80,7 +80,8 @@ def change_log(model_instance: "ModelInstance", attr_name: str):
     old_value = _get_change_value(getattr(model_instance.instance, attr_name))
     yield
     new_value = _get_change_value(getattr(model_instance.instance, attr_name))
-    if old_value != new_value:
+    if old_value != new_value or model_instance.environment.import_mode:
+
         if isinstance(old_value, set):
             model_instance.metadata.changes[attr_name] = {
                 "old_items": old_value,

nautobot_design_builder/jobs.py

@@ -1,6 +1,6 @@
 """Generic Design Builder Jobs."""
 
-from nautobot.extras.jobs import Job, MultiObjectVar
+from nautobot.extras.jobs import Job, MultiObjectVar, BooleanVar
 
 from .logging import get_logger
 from .models import Deployment
@@ -17,6 +17,10 @@ class DeploymentDecommissioning(Job):
         query_params={"status": "active"},
         description="Design Deployments to decommission.",
     )
+    delete = BooleanVar(
+        description="Actually delete the objects, not just their link to the design delpoyment.",
+        default=True,
+    )
 
     class Meta:  # pylint: disable=too-few-public-methods
         """Meta class."""
@@ -27,14 +31,27 @@ class Meta:  # pylint: disable=too-few-public-methods
     def run(self, data, commit):
         """Execute Decommissioning job."""
         deployments = data["deployments"]
+        delete = data["delete"]
+
         self.log_info(
             message=f"Starting decommissioning of design deployments: {', '.join([instance.name for instance in deployments])}",
         )
 
         for deployment in deployments:
-            self.log_info(obj=deployment, message="Working on resetting objects for this Design Deployment...")
-            deployment.decommission(local_logger=get_logger(__name__, self.job_result))
-            self.log_success(f"{deployment} has been successfully decommissioned from Nautobot.")
+            if delete:
+                message = "Working on deleting objects for this Design Deployment."
+            else:
+                message = "Working on unlinking objects from this Design Deployment."
+            self.log_info(obj=deployment, message=message)
+
+            deployment.decommission(local_logger=get_logger(__name__, self.job_result), delete=delete)
+
+            if delete:
+                message = f"{deployment} has been successfully decommissioned from Nautobot."
+            else:
+                message = f"Objects have been successfully unlinked from {deployment}."
+
+            self.log_success(message)
 
 
 jobs = (DeploymentDecommissioning,)

nautobot_design_builder/models.py

@@ -1,7 +1,8 @@
 """Collection of models that DesignBuilder uses to track design implementations."""
 
 import logging
-from typing import List
+from typing import List, Optional
+from uuid import UUID
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes import fields as ct_fields
 from django.core.exceptions import ValidationError, ObjectDoesNotExist
@@ -228,7 +229,7 @@ def __str__(self):
         """Stringify instance."""
         return f"{self.design.name} - {self.name}"
 
-    def decommission(self, *object_ids, local_logger=logger):
+    def decommission(self, *object_ids, local_logger=logger, delete=True):
         """Decommission a design instance.
 
         This will reverse the change records for the design instance and
@@ -240,7 +241,10 @@ def decommission(self, *object_ids, local_logger=logger):
         # Iterate the change sets in reverse order (most recent first) and
         # revert each change set.
         for change_set in self.change_sets.filter(active=True).order_by("-last_updated"):
-            change_set.revert(*object_ids, local_logger=local_logger)
+            if delete:
+                change_set.revert(*object_ids, local_logger=local_logger)
+            else:
+                change_set.deactivate()
 
         if not object_ids:
             content_type = ContentType.objects.get_for_model(Deployment)
@@ -343,7 +347,7 @@ def _next_index(self):
         setattr(self, "_index", index)
         return index
 
-    def log(self, model_instance):
+    def log(self, model_instance, import_mode: bool):
         """Log changes to a model instance.
 
         This will log the differences between a model instance's
@@ -354,6 +358,7 @@ def log(self, model_instance):
 
         Args:
             model_instance: Model instance to log changes.
+            import_mode: Boolean used to import design objects already present in the database.
         """
         instance = model_instance.instance
         content_type = ContentType.objects.get_for_model(instance)
@@ -368,13 +373,50 @@ def log(self, model_instance):
             entry.changes.update(model_instance.metadata.changes)
             entry.save()
         except ChangeRecord.DoesNotExist:
-            entry = self.records.create(
-                _design_object_type=content_type,
-                _design_object_id=instance.id,
-                changes=model_instance.metadata.changes,
-                full_control=model_instance.metadata.created,
-                index=self._next_index(),
-            )
+            entry_parameters = {
+                "_design_object_type": content_type,
+                "_design_object_id": instance.id,
+                "changes": model_instance.metadata.changes,
+                "full_control": model_instance.metadata.created,
+                "index": self._next_index(),
+            }
+            # Deferred import as otherwise Nautobot doesn't start
+            from .design import ModelMetadata  # pylint: disable=import-outside-toplevel,cyclic-import
+
+            # Path when not importing, either because it's not enabled or the action is not supported for importing.
+            if not import_mode or model_instance.metadata.action not in ModelMetadata.IMPORTABLE_ACTION_CHOICES:
+                entry = self.records.create(**entry_parameters)
+                return entry
+
+            # When we have intention to claim ownership (i.e. the action is CREATE_OR_UPDATE) we first try to obtain
+            # `full_control` over the object, thus pretending that we have created it.
+            # If the object is already owned with full_control by another Design Deployment,
+            # we acknowledge it and set `full_control` to `False`.
+            # TODO: Shouldn't this change record object also need to be active?
+            change_records_for_instance = ChangeRecord.objects.filter_by_design_object_id(_design_object_id=instance.id)
+            if model_instance.metadata.action == ModelMetadata.CREATE_OR_UPDATE:
+                entry_parameters["full_control"] = not change_records_for_instance.filter(full_control=True).exists()
+
+            # When we don't want to assume full control, make sure we don't try to own any of the query filter values.
+            # We do this by removing any query filter values from the `changes` dictionary, which is the structure that
+            # defines which attributes are owned by the deployment.
+            if not entry_parameters["full_control"]:
+                for attribute in model_instance.metadata.query_filter_values:
+                    entry_parameters["changes"].pop(attribute, None)
+
+            # Check if any owned attributes exist that conflict with the changes for this instance.
+            # We do this by iterating over all change records that exist for this instance, ...
+            for record in change_records_for_instance:
+                # ...iterating over all attributes in those instances changes...
+                for attribute in record.changes:
+                    # ...and, finally, by raising an error if any of those overlap with those attributes that we are
+                    # trying to own by importing the object.
+                    if attribute in entry_parameters["changes"]:
+                        raise ValueError(  # pylint: disable=raise-missing-from
+                            f"The {attribute} attribute for {instance} is already owned by Design Deployment {record.change_set.deployment}"
+                        )
+
+            entry = self.records.create(**entry_parameters)
         return entry
 
     def revert(self, *object_ids, local_logger: logging.Logger = logger):
@@ -432,6 +474,14 @@ def __sub__(self, other: "ChangeSet"):
             .values_list("_design_object_id", flat=True)
         )
 
+    def deactivate(self):
+        """Mark the change_set and its records as not active."""
+        self.active = False
+        for change_set_record in self.records.all():
+            change_set_record.active = False
+            change_set_record.save()
+        self.save()
+
 
 class ChangeRecordQuerySet(RestrictedQuerySet):
     """Queryset for `ChangeRecord` objects."""
@@ -500,6 +550,22 @@ def design_objects(self, deployment: "Deployment"):
         }
         return self.filter(id__in=design_object_ids.values())
 
+    def filter_by_design_object_id(self, _design_object_id: UUID, full_control: Optional[bool] = None):
+        """Lookup all the active records for a design object ID and an full_control.
+
+        Args:
+            _design_object_id (UUID): The design object UUID.
+            full_control (type, optional): Include the full_control filter. Defaults to None.
+
+        Returns:
+            Query set matching the options.
+        """
+        if full_control is not None:
+            queryset = self.filter(_design_object_id=_design_object_id, active=True, full_control=full_control)
+        else:
+            queryset = self.filter(_design_object_id=_design_object_id, active=True)
+        return queryset.exclude_decommissioned()
+
 
 class ChangeRecord(BaseModel):
     """A single entry in the change set for exactly 1 object.

nautobot_design_builder/tests/designs/templates/simple_design_1.yaml.j2

@@ -0,0 +1,4 @@
+---
+manufacturers:
+    - "!create_or_update:name": "Test Manufacturer"
+      "description": "Test description"

nautobot_design_builder/tests/designs/templates/simple_design_4.yaml.j2

@@ -0,0 +1,19 @@
+---
+manufacturers:
+  - "!create_or_update:name": "Test Manufacturer"
+    "description": "Test description"
+device_roles:
+  - "!create_or_update:name": "Switch"
+    "description": "Test description"
+device_types:
+  - "!create_or_update:model": "Test Device Type"
+    "manufacturer__name": "Test Manufacturer"
+sites:
+  - "!create_or_update:name": "Test Site"
+    "status__name": "Active"
+devices:
+  - "!create_or_update:name": "Test Device"
+    "device_role__name": "Switch"
+    "device_type__model": "Test Device Type"
+    "site__name": "Test Site"
+    "status__name": "Active"

nautobot_design_builder/tests/designs/templates/simple_design_5.yaml.j2

@@ -0,0 +1,4 @@
+---
+manufacturers:
+    - "!update:name": "Test Manufacturer"
+      "description": "Test description"