Merge pull request #2 from nautobot/min-max-rules

added min max rules model, api, and tests
nautobot · May 28, 2021 · c517015 · c517015
2 parents c939417 + 7f3c6bc
commit c517015
Show file tree

Hide file tree

Showing 29 changed files with 1,841 additions and 369 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -12,10 +12,11 @@ python:
   - 3.6
   - 3.7
   - 3.8
+  - 3.9
 
 env:
   matrix:
-    - "NAUTOBOT_VER=1.0.0"
+    - "NAUTOBOT_VER=1.0.2"
 
 services:
   - "docker"
@@ -37,7 +38,7 @@ jobs:
       script:
         - "invoke black"
         - "invoke bandit"
-        - "invoke pydocstyle"
+        # - "invoke pydocstyle"
         - "invoke flake8"
         - "invoke yamllint"
         # - "invoke pylint"

diff --git a/development/Dockerfile b/development/Dockerfile
@@ -1,32 +1,18 @@
 
-ARG python_ver=3.7
-FROM python:${python_ver}
+ARG PYTHON_VER
+ARG NAUTOBOT_VER
+FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER}
 
-ARG nautobot_ver=v1.0.0b1
-ENV PYTHONUNBUFFERED 1
-
-RUN mkdir /prom_cache
-ENV prometheus_multiproc_dir /prom_cache
-
-RUN mkdir -p /opt/nautobot
-
-RUN pip install --upgrade pip\
-  && pip install poetry
+WORKDIR /source
 
-# -------------------------------------------------------------------------------------
-# Install Nautobot
-# -------------------------------------------------------------------------------------
-COPY packages/nautobot-1.0.0b1-py3-none-any.whl /tmp
-RUN pip install /tmp/nautobot-1.0.0b1-py3-none-any.whl
+# Copy in only pyproject.toml/poetry.lock to help with caching this layer if no updates to dependencies
+COPY poetry.lock pyproject.toml /source/
+# --no-root declares not to install the project package since we're wanting to take advantage of caching dependency installation
+# and the project is copied in and installed after this step
+RUN poetry install --no-interaction --no-ansi --no-root
 
-# -------------------------------------------------------------------------------------
-# Install Nautobot Plugin
-# -------------------------------------------------------------------------------------
-RUN mkdir -p /source
-WORKDIR /source
+# Copy in the rest of the source code and install local Nautobot plugin
 COPY . /source
-RUN poetry config virtualenvs.create false \
-  && poetry install --no-interaction --no-ansi
+RUN poetry install --no-interaction --no-ansi
 
-ENV NAUTOBOT_CONFIG /opt/nautobot/configuration.py
-WORKDIR /opt/nautobot/
+COPY development/nautobot_config.py /opt/nautobot/nautobot_config.py
diff --git a/development/dev.env b/development/dev.env
@@ -1,20 +1,24 @@
-ALLOWED_HOSTS=*
-CHANGELOG_RETENTION=0
-DB_HOST=postgres
-DB_NAME=nautobot
-DB_PASSWORD=decinablesprewad
-DB_USER=nautobot
-HIDE_RESTRICTED_UI=True
-MAX_PAGE_SIZE=0
-NAPALM_TIMEOUT=5
-NAUTOBOT_CONFIG=/etc/nautobot/nautobot_config.py
+NAUTOBOT_ALLOWED_HOSTS=*
+NAUTOBOT_CHANGELOG_RETENTION=0
+NAUTOBOT_CONFIG=/opt/nautobot/nautobot_config.py
+NAUTOBOT_DB_HOST=postgres
+NAUTOBOT_DB_NAME=nautobot
+NAUTOBOT_DB_PASSWORD=decinablesprewad
+NAUTOBOT_DB_USER=nautobot
+NAUTOBOT_MAX_PAGE_SIZE=0
+NAUTOBOT_NAPALM_TIMEOUT=5
+NAUTOBOT_REDIS_HOST=redis
+NAUTOBOT_REDIS_PASSWORD=decinablesprewad
+NAUTOBOT_REDIS_PORT=6379
+# Uncomment REDIS_SSL if using SSL
+# NAUTOBOT_REDIS_SSL=True
+NAUTOBOT_SECRET_KEY=012345678901234567890123456789012345678901234567890123456789
+
+# Needed for Postgres should match the values for Nautobot above
 PGPASSWORD=decinablesprewad
 POSTGRES_DB=nautobot
 POSTGRES_PASSWORD=decinablesprewad
 POSTGRES_USER=nautobot
-REDIS_HOST=redis
+
+# Needed for Redis should match the values for Nautobot above
 REDIS_PASSWORD=decinablesprewad
-REDIS_PORT=6379
-# Uncomment REDIS_SSL if using SSL
-# REDIS_SSL=True
-SECRET_KEY=012345678901234567890123456789012345678901234567890123456789
diff --git a/development/docker-compose.yml b/development/docker-compose.yml
@@ -1,53 +1,59 @@
+
 ---
-version: "3"
-services:
-  nautobot:
+  x-nautobot-build: &nautobot-build
     build:
+      args:
+        NAUTOBOT_VER: "${NAUTOBOT_VER}"
+        PYTHON_VER: "${PYTHON_VER}"
       context: "../"
       dockerfile: "development/Dockerfile"
-    image: "nautobot-data-validation-engine/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}"
-    command: >
-      sh -c "nautobot-server migrate &&
-             nautobot-server runserver 0.0.0.0:8000"
-    ports:
-      - "8000:8000"
-    depends_on:
-      - "postgres"
-      - "redis"
+  x-nautobot-base: &nautobot-base
+    image: "nautobot_data_validation_engine/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}"
     env_file:
-      - "./dev.env"
-    volumes:
-      - "./nautobot_config.py:/etc/nautobot/nautobot_config.py"
-      - "../:/source"
+      - "dev.env"
     tty: true
-  worker:
-    build:
-      context: "../"
-      dockerfile: "development/Dockerfile"
-    image: "nautobot-data-validation-engine/nautobot:${NAUTOBOT_VER}-py${PYTHON_VER}"
-    command: >
-      sh -c "nautobot-server rqworker"
-    depends_on:
-      - "nautobot"
-    env_file:
-      - "./dev.env"
-    volumes:
-      - "./nautobot_config.py:/etc/nautobot/nautobot_config.py"
-      - "../nautobot_data_validation_engine:/source/nautobot_data_validation_engine"
-    tty: true
-  postgres:
-    image: "postgres:12"
-    env_file:
-      - "./dev.env"
-    volumes:
-      - "pgdata_nautobot_data_validation_engine:/var/lib/postgresql/data"
-  redis:
-    image: "redis:5-alpine"
-    command:
-      - "sh"
-      - "-c"  # this is to evaluate the $REDIS_PASSWORD from the env
-      - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD"  ## $$ because of docker-compose
-    env_file:
-      - "./dev.env"
-volumes:
-  pgdata_nautobot_data_validation_engine: {}
+
+  version: "3.4"
+  services:
+    nautobot:
+      command: "nautobot-server runserver 0.0.0.0:8080 --insecure"
+      volumes:
+        - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
+        - "../:/source"
+      ports:
+        - "8080:8080"
+      depends_on:
+        - "postgres"
+        - "redis"
+      <<: *nautobot-build
+      <<: *nautobot-base
+    worker:
+      entrypoint: "nautobot-server rqworker"
+      volumes:
+        - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
+        - "../:/source"
+      depends_on:
+        - "nautobot"
+      healthcheck:
+        disable: true
+      <<: *nautobot-base
+    postgres:
+      image: "postgres:13-alpine"
+      env_file:
+        - "dev.env"
+      volumes:
+        - "pgdata_nautobot_data_validation_engine:/var/lib/postgresql/data"
+      ports:
+        - "5432:5432"
+    redis:
+      image: "redis:6-alpine"
+      command:
+        - "sh"
+        - "-c" # this is to evaluate the $REDIS_PASSWORD from the env
+        - "redis-server --appendonly yes --requirepass $$REDIS_PASSWORD"
+      env_file:
+        - "dev.env"
+      ports:
+        - "6379:6379"
+  volumes:
+    pgdata_nautobot_data_validation_engine:  # yamllint disable-line rule:empty-values
diff --git a/development/nautobot_config.py b/development/nautobot_config.py
@@ -1,16 +1,19 @@
-"""Nautobot configuration file."""
+"""Nautobot development configuration file."""
 import os
 import sys
 
-ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(" ")
+from nautobot.core.settings import *
+from nautobot.core.settings_funcs import is_truthy
+
+ALLOWED_HOSTS = os.environ.get("NAUTOBOT_ALLOWED_HOSTS", "").split(" ")
 
 DATABASES = {
     "default": {
-        "NAME": os.environ.get("DB_NAME", "nautobot"),
-        "USER": os.environ.get("DB_USER", ""),
-        "PASSWORD": os.environ.get("DB_PASSWORD", ""),
-        "HOST": os.environ.get("DB_HOST", "localhost"),
-        "PORT": os.environ.get("DB_PORT", ""),
+        "NAME": os.environ.get("NAUTOBOT_DB_NAME", "nautobot"),
+        "USER": os.environ.get("NAUTOBOT_DB_USER", ""),
+        "PASSWORD": os.environ.get("NAUTOBOT_DB_PASSWORD", ""),
+        "HOST": os.environ.get("NAUTOBOT_DB_HOST", "localhost"),
+        "PORT": os.environ.get("NAUTOBOT_DB_PORT", ""),
         "CONN_MAX_AGE": 300,
         "ENGINE": "django.db.backends.postgresql",
     }
@@ -20,54 +23,94 @@
 
 LOG_LEVEL = "DEBUG" if DEBUG else "INFO"
 
-LOGGING = {
-    "version": 1,
-    "disable_existing_loggers": False,
-    "formatters": {
-        "normal": {
-            "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s :\n  %(message)s",
-            "datefmt": "%H:%M:%S",
+TESTING = len(sys.argv) > 1 and sys.argv[1] == "test"
+
+# Verbose logging during normal development operation, but quiet logging during unit test execution
+if not TESTING:
+    LOGGING = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "normal": {
+                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)s :\n  %(message)s",
+                "datefmt": "%H:%M:%S",
+            },
+            "verbose": {
+                "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() :\n  %(message)s",
+                "datefmt": "%H:%M:%S",
+            },
         },
-        "verbose": {
-            "format": "%(asctime)s.%(msecs)03d %(levelname)-7s %(name)-20s %(filename)-15s %(funcName)30s() :\n  %(message)s",
-            "datefmt": "%H:%M:%S",
+        "handlers": {
+            "normal_console": {
+                "level": "INFO",
+                "class": "rq.utils.ColorizingStreamHandler",
+                "formatter": "normal",
+            },
+            "verbose_console": {
+                "level": "DEBUG",
+                "class": "rq.utils.ColorizingStreamHandler",
+                "formatter": "verbose",
+            },
         },
-    },
-    "handlers": {
-        "normal_console": {"level": "INFO", "class": "rq.utils.ColorizingStreamHandler", "formatter": "normal"},
-        "verbose_console": {"level": "DEBUG", "class": "rq.utils.ColorizingStreamHandler", "formatter": "verbose"},
-    },
-    "loggers": {
-        "django": {"handlers": ["normal_console"], "level": "INFO"},
-        "nautobot": {"handlers": ["verbose_console" if DEBUG else "normal_console"], "level": LOG_LEVEL},
-        "rq.worker": {"handlers": ["verbose_console" if DEBUG else "normal_console"], "level": LOG_LEVEL},
-    },
-}
+        "loggers": {
+            "django": {"handlers": ["normal_console"], "level": "INFO"},
+            "nautobot": {
+                "handlers": ["verbose_console" if DEBUG else "normal_console"],
+                "level": LOG_LEVEL,
+            },
+            "rq.worker": {
+                "handlers": ["verbose_console" if DEBUG else "normal_console"],
+                "level": LOG_LEVEL,
+            },
+        },
+    }
+
+
+# Redis variables
+REDIS_HOST = os.getenv("NAUTOBOT_REDIS_HOST", "localhost")
+REDIS_PORT = os.getenv("NAUTOBOT_REDIS_PORT", 6379)
+REDIS_PASSWORD = os.getenv("NAUTOBOT_REDIS_PASSWORD", "")
 
-REDIS = {
-    "caching": {
-        "HOST": os.environ.get("REDIS_HOST", "redis"),
-        "PORT": int(os.environ.get("REDIS_PORT", 6379)),
-        "PASSWORD": os.environ.get("REDIS_PASSWORD", ""),
-        "DATABASE": 1,
-        "SSL": bool(os.environ.get("REDIS_SSL", False)),
-    },
-    "tasks": {
-        "HOST": os.environ.get("REDIS_HOST", "redis"),
-        "PORT": int(os.environ.get("REDIS_PORT", 6379)),
-        "PASSWORD": os.environ.get("REDIS_PASSWORD", ""),
-        "DATABASE": 0,
-        "SSL": bool(os.environ.get("REDIS_SSL", False)),
-    },
+# Check for Redis SSL
+REDIS_SCHEME = "redis"
+REDIS_SSL = is_truthy(os.environ.get("NAUTOBOT_REDIS_SSL", False))
+if REDIS_SSL:
+    REDIS_SCHEME = "rediss"
+
+# The django-redis cache is used to establish concurrent locks using Redis. The
+# django-rq settings will use the same instance/database by default.
+#
+# This "default" server is now used by RQ_QUEUES.
+# >> See: nautobot.core.settings.RQ_QUEUES
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": f"{REDIS_SCHEME}://{REDIS_HOST}:{REDIS_PORT}/0",
+        "TIMEOUT": 300,
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            "PASSWORD": REDIS_PASSWORD,
+        },
+    }
 }
 
-SECRET_KEY = os.environ.get("SECRET_KEY", "")
+# RQ_QUEUES is not set here because it just uses the default that gets imported
+# up top via `from nautobot.core.settings import *`.
+
+# REDIS CACHEOPS
+CACHEOPS_REDIS = f"{REDIS_SCHEME}://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/1"
+
+HIDE_RESTRICTED_UI = os.environ.get("HIDE_RESTRICTED_UI", False)
+
+SECRET_KEY = os.environ.get("NAUTOBOT_SECRET_KEY", "")
 
 # Django Debug Toolbar
-TESTING = len(sys.argv) > 1 and sys.argv[1] == "test"
 DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": lambda _request: DEBUG and not TESTING}
-HIDE_RESTRICTED_UI = os.environ.get("HIDE_RESTRICTED_UI", False)
 
-PLUGINS = [
-    "nautobot_data_validation_engine",
-]
+if "debug_toolbar" not in INSTALLED_APPS:
+    INSTALLED_APPS.append("debug_toolbar")
+if "debug_toolbar.middleware.DebugToolbarMiddleware" not in MIDDLEWARE:
+    MIDDLEWARE.insert(0, "debug_toolbar.middleware.DebugToolbarMiddleware")
+
+# Enable installed plugins. Add the name of each plugin to the list.
+PLUGINS = ["nautobot_data_validation_engine"]
diff --git a/nautobot_data_validation_engine/api/__init__.py b/nautobot_data_validation_engine/api/__init__.py