Feature delices v2

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ on: # yamllint disable-line rule:truthy rule:comments
   pull_request: ~
 
 env:
-  PLUGIN_NAME: "nautobot-app-design-builder"
+  APP_NAME: "nautobot-app-design-builder"
 
 jobs:
   black:
@@ -24,7 +24,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: black"
         run: "poetry run invoke black"
   bandit:
@@ -35,20 +35,31 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: bandit"
         run: "poetry run invoke bandit"
-  pydocstyle:
+  ruff:
     runs-on: "ubuntu-22.04"
     env:
       INVOKE_NAUTOBOT_DESIGN_BUILDER_LOCAL: "True"
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
-      - name: "Linting: pydocstyle"
-        run: "poetry run invoke pydocstyle"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
+      - name: "Linting: ruff"
+        run: "poetry run invoke ruff"
+  check-docs-build:
+    runs-on: "ubuntu-22.04"
+    env:
+      INVOKE_NAUTOBOT_DESIGN_BUILDER_LOCAL: "True"
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
+      - name: "Check Docs Build"
+        run: "poetry run invoke build-and-check-docs"
   flake8:
     runs-on: "ubuntu-22.04"
     env:
@@ -57,7 +68,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: flake8"
         run: "poetry run invoke flake8"
   poetry:
@@ -68,7 +79,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Checking: poetry lock file"
         run: "poetry run invoke lock --check"
   yamllint:
@@ -79,13 +90,13 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Linting: yamllint"
         run: "poetry run invoke yamllint"
-  pylint:
+  check-in-docker:
     needs:
       - "bandit"
-      - "pydocstyle"
+      - "ruff"
       - "flake8"
       - "poetry"
       - "yamllint"
@@ -95,15 +106,15 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ["3.11"]
-        nautobot-version: ["2.2"]
+        nautobot-version: ["stable"]
     env:
       INVOKE_NAUTOBOT_DESIGN_BUILDER_PYTHON_VER: "${{ matrix.python-version }}"
       INVOKE_NAUTOBOT_DESIGN_BUILDER_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Set up Docker Buildx"
         id: "buildx"
         uses: "docker/setup-buildx-action@v3"
@@ -114,7 +125,7 @@ jobs:
           context: "./"
           push: false
           load: true
-          tags: "${{ env.PLUGIN_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          tags: "${{ env.APP_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
           file: "./development/Dockerfile"
           cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
           cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
@@ -125,60 +136,23 @@ jobs:
         run: "cp development/creds.example.env development/creds.env"
       - name: "Linting: pylint"
         run: "poetry run invoke pylint"
-  check-migrations:
-    needs:
-      - "bandit"
-      - "pydocstyle"
-      - "flake8"
-      - "poetry"
-      - "yamllint"
-      - "black"
-    runs-on: "ubuntu-22.04"
-    strategy:
-      fail-fast: true
-      matrix:
-        python-version: ["3.11"]
-        nautobot-version: ["2.2"]
-    env:
-      INVOKE_NAUTOBOT_DESIGN_BUILDER_PYTHON_VER: "${{ matrix.python-version }}"
-      INVOKE_NAUTOBOT_DESIGN_BUILDER_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
-    steps:
-      - name: "Check out repository code"
-        uses: "actions/checkout@v4"
-      - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
-      - name: "Set up Docker Buildx"
-        id: "buildx"
-        uses: "docker/setup-buildx-action@v3"
-      - name: "Build"
-        uses: "docker/build-push-action@v5"
-        with:
-          builder: "${{ steps.buildx.outputs.name }}"
-          context: "./"
-          push: false
-          load: true
-          tags: "${{ env.PLUGIN_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
-          file: "./development/Dockerfile"
-          cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
-          cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
-          build-args: |
-            NAUTOBOT_VER=${{ matrix.nautobot-version }}
-            PYTHON_VER=${{ matrix.python-version }}
-      - name: "Copy credentials"
-        run: "cp development/creds.example.env development/creds.env"
+      - name: "Checking: App Config"
+        run: "poetry run invoke validate-app-config"
       - name: "Checking: migrations"
         run: "poetry run invoke check-migrations"
   unittest:
     needs:
-      - "pylint"
-      - "check-migrations"
+      - "check-in-docker"
     strategy:
       fail-fast: true
       matrix:
         python-version: ["3.8", "3.11"]
         db-backend: ["postgresql"]
         nautobot-version: ["stable"]
         include:
+          - python-version: "3.11"
+            db-backend: "postgresql"
+            nautobot-version: "stable"
           - python-version: "3.11"
             db-backend: "mysql"
             nautobot-version: "stable"
@@ -190,7 +164,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        uses: "networktocode/gh-action-setup-poetry-environment@v4"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
       - name: "Set up Docker Buildx"
         id: "buildx"
         uses: "docker/setup-buildx-action@v3"
@@ -201,7 +175,7 @@ jobs:
           context: "./"
           push: false
           load: true
-          tags: "${{ env.PLUGIN_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
+          tags: "${{ env.APP_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
           file: "./development/Dockerfile"
           cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
           cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}"
@@ -215,17 +189,35 @@ jobs:
         if: "matrix.db-backend == 'mysql'"
       - name: "Run Tests"
         run: "poetry run invoke unittest"
+  changelog:
+    if: |
+      contains(fromJson('["develop","ltm-1.6"]'), github.base_ref) &&
+      (github.head_ref != 'main')
+    runs-on: "ubuntu-22.04"
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+        with:
+          fetch-depth: "0"
+      - name: "Setup environment"
+        uses: "networktocode/gh-action-setup-poetry-environment@v6"
+      - name: "Check for changelog entry"
+        run: |
+          git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
+          poetry run towncrier check --compare-with origin/${{ github.base_ref }}
   publish_gh:
     needs:
       - "unittest"
     name: "Publish to GitHub"
     runs-on: "ubuntu-22.04"
     if: "startsWith(github.ref, 'refs/tags/v')"
+    env:
+      INVOKE_NAUTOBOT_DESIGN_BUILDER_LOCAL: "True"
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Set up Python"
-        uses: "actions/setup-python@v4"
+        uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
       - name: "Install Python Packages"
@@ -234,12 +226,16 @@ jobs:
         run: "echo RELEASE_VERSION=${GITHUB_REF:10} >> $GITHUB_ENV"
       - name: "Run Poetry Version"
         run: "poetry version $RELEASE_VERSION"
+      - name: "Install Dependencies (needed for mkdocs)"
+        run: "poetry install --no-root"
+      - name: "Build Documentation"
+        run: "poetry run invoke build-and-check-docs"
       - name: "Run Poetry Build"
         run: "poetry build"
       - name: "Upload binaries to release"
         uses: "svenstaro/upload-release-action@v2"
         with:
-          repo_token: "${{ secrets.GH_NAUTOBOT_BOT_TOKEN }}"
+          repo_token: "${{ secrets.NTC_GITHUB_TOKEN }}"  # use GH_NAUTOBOT_BOT_TOKEN for Nautobot Org repos.
           file: "dist/*"
           tag: "${{ github.ref }}"
           overwrite: true
@@ -250,11 +246,13 @@ jobs:
     name: "Push Package to PyPI"
     runs-on: "ubuntu-22.04"
     if: "startsWith(github.ref, 'refs/tags/v')"
+    env:
+      INVOKE_NAUTOBOT_DESIGN_BUILDER_LOCAL: "True"
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Set up Python"
-        uses: "actions/setup-python@v4"
+        uses: "actions/setup-python@v5"
         with:
           python-version: "3.11"
       - name: "Install Python Packages"
@@ -263,6 +261,10 @@ jobs:
         run: "echo RELEASE_VERSION=${GITHUB_REF:10} >> $GITHUB_ENV"
       - name: "Run Poetry Version"
         run: "poetry version $RELEASE_VERSION"
+      - name: "Install Dependencies (needed for mkdocs)"
+        run: "poetry install --no-root"
+      - name: "Build Documentation"
+        run: "poetry run invoke build-and-check-docs"
       - name: "Run Poetry Build"
         run: "poetry build"
       - name: "Push to PyPI"

README.md

@@ -15,6 +15,8 @@
 
 Design Builder is a Nautobot application for easily populating data within Nautobot using standardized design files. These design files are just Jinja templates that describe the Nautobot objects to be created or updated.
 
+It also introduces the concept of a design-oriented Source of Truth with a complete lifecycle management of the design deployments (i.e., an instantiation of a design with concrete input data). With this approach, the users of the application can not only create (or populate) data within Nautobot but also update or decommission it while enforcing data protection and dependency.
+
 ## Documentation
 
 Full documentation for this App can be found over on the [Nautobot Docs](https://docs.nautobot.com) website:

changes/.gitignore

@@ -0,0 +1 @@
+!.gitignore

changes/146.added

@@ -0,0 +1,2 @@
+Add a new mode that tracks design deployments providing a full lifecycle for design updates and decommissioning
+Provide data protection (optional) for data that has been created or modified by a design deployment.

development/app_config_schema.py

@@ -0,0 +1,65 @@
+"""App Config Schema Generator and Validator."""
+
+import json
+from importlib import import_module
+from os import getenv
+from pathlib import Path
+from urllib.parse import urlparse
+
+import jsonschema
+import toml
+from django.conf import settings
+from to_json_schema.to_json_schema import SchemaBuilder
+
+
+def _enrich_object_schema(schema, defaults, required):
+    schema["additionalProperties"] = False
+    for key, value in schema["properties"].items():
+        if required and key in required:
+            value["required"] = True
+        default_value = defaults and defaults.get(key, None)
+        if value["type"] == "object" and "properties" in value:
+            _enrich_object_schema(value, default_value, None)
+        elif default_value is not None:
+            value["default"] = default_value
+
+
+def _main():
+    pyproject = toml.loads(Path("pyproject.toml").read_text())
+    url = urlparse(pyproject["tool"]["poetry"]["repository"])
+    _, owner, repository = url.path.split("/")
+    package_name = pyproject["tool"]["poetry"]["packages"][0]["include"]
+    app_config = settings.PLUGINS_CONFIG[package_name]  # type: ignore
+    schema_path = Path(package_name) / "app-config-schema.json"
+    command = getenv("APP_CONFIG_SCHEMA_COMMAND", "")
+    if command == "generate":
+        schema = {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "$id": f"https://raw.githubusercontent.com/{owner}/{repository}/develop/{package_name}/app-config-schema.json",
+            "$comment": "TBD: Update $id, replace `develop` with the future release tag",
+            **SchemaBuilder().to_json_schema(app_config),  # type: ignore
+        }
+        app_config = import_module(package_name).config
+        _enrich_object_schema(schema, app_config.default_settings, app_config.required_settings)
+        schema_path.write_text(json.dumps(schema, indent=4) + "\n")
+        print(f"\n==================\nGenerated schema:\n\n{schema_path}\n")
+        print(
+            "WARNING: Review and edit the generated file before committing.\n"
+            "\n"
+            "Its content is inferred from:\n"
+            "\n"
+            "- The current configuration in `PLUGINS_CONFIG`\n"
+            "- `NautobotAppConfig.default_settings`\n"
+            "- `NautobotAppConfig.required_settings`"
+        )
+    elif command == "validate":
+        schema = json.loads(schema_path.read_text())
+        jsonschema.validate(app_config, schema)
+        print(
+            f"\n==================\nValidated configuration using the schema:\n{schema_path}\nConfiguration is valid."
+        )
+    else:
+        raise RuntimeError(f"Unknown command: {command}")
+
+
+_main()

development/nautobot_config.py

@@ -21,6 +21,9 @@
     if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE:  # noqa: F405
         MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware")  # noqa: F405
 
+if "nautobot_design_builder.middleware.GlobalRequestMiddleware" not in MIDDLEWARE:  # noqa: F405
+    MIDDLEWARE.insert(0, "nautobot_design_builder.middleware.GlobalRequestMiddleware")  # noqa: F405
+
 #
 # Misc. settings
 #
@@ -138,4 +141,19 @@
 if is_truthy(os.getenv("DESIGN_BUILDER_ENABLE_BGP", "False")):
     PLUGINS.append("nautobot_bgp_models")
 
-PLUGINS_CONFIG = {"design_builder": {"context_repository": os.getenv("DESIGN_BUILDER_CONTEXT_REPO_SLUG", None)}}
+
+def pre_decommission_hook_example(design_instance):
+    """Example decomission hook."""
+    return True, "Everything good!"
+
+
+PLUGINS_CONFIG = {
+    "nautobot_design_builder": {
+        "context_repository": os.getenv("DESIGN_BUILDER_CONTEXT_REPO_SLUG", None),
+        "pre_decommission_hook": pre_decommission_hook_example,
+        "protected_models": [("dcim", "region"), ("dcim", "device"), ("dcim", "interface")],
+        "protected_superuser_bypass": False,
+    }
+}
+
+STRICT_FILTERING = False