From 308a1b0d8b595231ef8f0a31f52e8b71d017d001 Mon Sep 17 00:00:00 2001
From: SG <mero.mero.guero@gmail.com>
Date: Thu, 26 Oct 2023 13:30:45 -0600
Subject: [PATCH] add option to auto-create catch-all netbox IPAM prefixes for
 private IP space (idaholab/Malcolm#279)

---
 Dockerfiles/netbox.Dockerfile            |  2 +
 config/netbox-common.env.example         |  2 +
 docs/kubernetes.md                       |  2 +
 docs/malcolm-hedgehog-e2e-iso-install.md |  2 +
 netbox/preload/prefixes_defaults.yml     |  6 +++
 netbox/preload/vrfs_defaults.yml         |  6 +++
 netbox/scripts/netbox_init.py            | 56 ++++++++++++++++++------
 netbox/supervisord.conf                  |  2 +
 scripts/install.py                       | 19 ++++++++
 9 files changed, 83 insertions(+), 14 deletions(-)
 create mode 100644 netbox/preload/prefixes_defaults.yml
 create mode 100644 netbox/preload/vrfs_defaults.yml

diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile
index 8fbb0a628..4d5dbc7fb 100644
--- a/Dockerfiles/netbox.Dockerfile
+++ b/Dockerfiles/netbox.Dockerfile
@@ -39,6 +39,7 @@ ARG NETBOX_DEVICETYPE_LIBRARY_PATH="/opt/netbox-devicetype-library"
 ARG NETBOX_DEFAULT_SITE=Malcolm
 ARG NETBOX_CRON=true
 ARG NETBOX_PRELOAD_PATH="/opt/netbox-preload"
+ARG NETBOX_PRELOAD_PREFIXES=false
 
 ENV NETBOX_PATH /opt/netbox
 ENV BASE_PATH netbox
@@ -46,6 +47,7 @@ ENV NETBOX_DEVICETYPE_LIBRARY_PATH $NETBOX_DEVICETYPE_LIBRARY_PATH
 ENV NETBOX_DEFAULT_SITE $NETBOX_DEFAULT_SITE
 ENV NETBOX_CRON $NETBOX_CRON
 ENV NETBOX_PRELOAD_PATH $NETBOX_PRELOAD_PATH
+ENV NETBOX_PRELOAD_PREFIXES $NETBOX_PRELOAD_PREFIXES
 
 ADD netbox/patch/* /tmp/netbox-patches/
 
diff --git a/config/netbox-common.env.example b/config/netbox-common.env.example
index 882cc64ae..0caba9062 100644
--- a/config/netbox-common.env.example
+++ b/config/netbox-common.env.example
@@ -3,6 +3,8 @@
 # The name of the default "site" to be created upon NetBox initialization, and to be queried
 #   for enrichment (see LOGSTASH_NETBOX_ENRICHMENT)
 NETBOX_DEFAULT_SITE=Malcolm
+# Whether or not to create catch-all VRFs/IP Prefixes for private IP space
+NETBOX_PRELOAD_PREFIXES=false
 # Whether to disable Malcolm's NetBox instance ('true') or not ('false')
 NETBOX_DISABLED=true
 NETBOX_POSTGRES_DISABLED=true
diff --git a/docs/kubernetes.md b/docs/kubernetes.md
index 3f8c9d3c5..91294d6c1 100644
--- a/docs/kubernetes.md
+++ b/docs/kubernetes.md
@@ -430,6 +430,8 @@ Should Malcolm automatically populate NetBox inventory based on observed network
 
 Specify default NetBox site name: Malcolm
 
+Should Malcolm create "catch-all" prefixes for private IP address space? (y / N): n
+
 Enable dark mode for OpenSearch Dashboards? (Y / n): y
 
 Malcolm has been installed to /home/user/Malcolm. See README.md for more information.
diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md
index 492d8fe1f..280698e90 100644
--- a/docs/malcolm-hedgehog-e2e-iso-install.md
+++ b/docs/malcolm-hedgehog-e2e-iso-install.md
@@ -255,6 +255,8 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest
     - Answer **Y** to [populate the NetBox inventory](asset-interaction-analysis.md#NetBoxPopPassive) based on observed network traffic. Autopopulation is **not** recommended: [manual inventory population](asset-interaction-analysis.md#NetBoxPopManual) is the preferred method to create an accurate representation of the intended network design.
 * **Specify default NetBox site name**
     - NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/); this default site name will be used as a query parameter for these enrichment lookups.
+* **Should Malcolm create "catch-all" prefixes for private IP address space?**
+    - Answer **Y** to automatically create "catch-all" NetBox prefixes for private IP address space (i.e., one each for `10.0.0.0/8`, `172.16.0.0/12`, and `192.168.0.0/16`, respectively). This is not recommended for networks with more than one subnet.
 * **Should Malcolm capture live network traffic?**
     - Malcolm itself can perform [live analysis](live-analysis.md#LocalPCAP) of traffic it sees on another network interface (ideally not the same one used for its management). Answer **no** to this question in installations where Hedgehog Linux will be handling all network traffic capture. If users want Malcolm to observe and capture traffic instead of, or in addition to, a sensor running Hedgehog Linux, they should answer **yes** enable life traffic analysis using default settings, or select **customize** to proceed to answer the following related questions individually.
     - **Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?**
diff --git a/netbox/preload/prefixes_defaults.yml b/netbox/preload/prefixes_defaults.yml
new file mode 100644
index 000000000..0e7d4c736
--- /dev/null
+++ b/netbox/preload/prefixes_defaults.yml
@@ -0,0 +1,6 @@
+- prefix: 10.0.0.0/8
+  vrf: Private IP Space (10.0.0.0/8)
+- prefix: 172.16.0.0/12
+  vrf: Private IP Space (172.16.0.0/12)
+- prefix: 10.0.0.0/8
+  vrf: Private IP Space (192.168.0.0/16)
diff --git a/netbox/preload/vrfs_defaults.yml b/netbox/preload/vrfs_defaults.yml
new file mode 100644
index 000000000..866439507
--- /dev/null
+++ b/netbox/preload/vrfs_defaults.yml
@@ -0,0 +1,6 @@
+- enforce_unique: true
+  name: Private IP Space (10.0.0.0/8)
+- enforce_unique: true
+  name: Private IP Space (172.16.0.0/12)
+- enforce_unique: true
+  name: Private IP Space (192.168.0.0/16)
diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py
index 0e2f500fe..a41063f9b 100755
--- a/netbox/scripts/netbox_init.py
+++ b/netbox/scripts/netbox_init.py
@@ -12,11 +12,14 @@
 import pynetbox
 import randomcolor
 import re
+import shutil
 import sys
+import tempfile
 import time
 import malcolm_utils
 
 from collections.abc import Iterable
+from distutils.dir_util import copy_tree
 from datetime import datetime
 from slugify import slugify
 from netbox_library_import import import_library
@@ -238,6 +241,16 @@ def main():
         required=False,
         help="Directory containing netbox-initializers files to preload",
     )
+    parser.add_argument(
+        '--preload-prefixes',
+        dest='preloadPrefixes',
+        type=malcolm_utils.str2bool,
+        metavar="true|false",
+        nargs='?',
+        const=True,
+        default=malcolm_utils.str2bool(os.getenv('NETBOX_PRELOAD_PREFIXES', default='False')),
+        help="Preload IPAM VRFs/IP Prefixes for private IP space",
+    )
     try:
         parser.error = parser.exit
         args = parser.parse_args()
@@ -642,20 +655,35 @@ def main():
     if os.path.isfile(netboxVenvPy) and os.path.isfile(manageScript) and os.path.isdir(args.preloadDir):
         try:
             with malcolm_utils.pushd(os.path.dirname(manageScript)):
-                retcode, output = malcolm_utils.run_process(
-                    [
-                        netboxVenvPy,
-                        os.path.basename(manageScript),
-                        "load_initializer_data",
-                        "--path",
-                        args.preloadDir,
-                    ],
-                    logger=logging,
-                )
-                if retcode == 0:
-                    logging.debug(f"netbox-initializers: {retcode} {output}")
-                else:
-                    logging.error(f"Error processing netbox-initializers: {retcode} {output}")
+                # make a local copy of the YMLs to preload
+                with tempfile.TemporaryDirectory() as tmpPreloadDir:
+                    copy_tree(args.preloadDir, tmpPreloadDir)
+
+                    # only preload catch-all VRFs and IP Prefixes if explicitly specified and they don't already exist
+                    if args.preloadPrefixes:
+                        for loadType in ('vrfs', 'prefixes'):
+                            defaultFileName = os.path.join(tmpPreloadDir, f'{loadType}_defaults.yml')
+                            loadFileName = os.path.join(tmpPreloadDir, f'{loadType}.yml')
+                            if os.path.isfile(defaultFileName) and (not os.path.isfile(loadFileName)):
+                                try:
+                                    shutil.copyfile(defaultFileName, loadFileName)
+                                except Exception:
+                                    pass
+
+                    retcode, output = malcolm_utils.run_process(
+                        [
+                            netboxVenvPy,
+                            os.path.basename(manageScript),
+                            "load_initializer_data",
+                            "--path",
+                            tmpPreloadDir,
+                        ],
+                        logger=logging,
+                    )
+                    if retcode == 0:
+                        logging.debug(f"netbox-initializers: {retcode} {output}")
+                    else:
+                        logging.error(f"Error processing netbox-initializers: {retcode} {output}")
 
         except Exception as e:
             logging.error(f"{type(e).__name__} processing netbox-initializers: {e}")
diff --git a/netbox/supervisord.conf b/netbox/supervisord.conf
index 44e997720..8fe938f56 100644
--- a/netbox/supervisord.conf
+++ b/netbox/supervisord.conf
@@ -39,6 +39,8 @@ command=/opt/netbox/venv/bin/python /usr/local/bin/netbox_init.py
   --token "%(ENV_SUPERUSER_API_TOKEN)s"
   --net-map /usr/local/share/net-map.json
   --library "%(ENV_NETBOX_DEVICETYPE_LIBRARY_PATH)s"
+  --preload "%(ENV_NETBOX_PRELOAD_PATH)s"
+  --preload-prefixes %(ENV_NETBOX_PRELOAD_PREFIXES)s
 autostart=true
 autorestart=false
 startsecs=0
diff --git a/scripts/install.py b/scripts/install.py
index 09285cef0..ad3b6b022 100755
--- a/scripts/install.py
+++ b/scripts/install.py
@@ -1263,6 +1263,10 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
         )
         if len(netboxSiteName) == 0:
             netboxSiteName = 'Malcolm'
+        netboxPreloadPrefixes = netboxEnabled and InstallerYesOrNo(
+            'Should Malcolm create "catch-all" prefixes for private IP address space?',
+            default=args.netboxPreloadPrefixes,
+        )
 
         # input packet capture parameters
         pcapNetSniff = False
@@ -1511,6 +1515,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path):
                 'NETBOX_DISABLED',
                 TrueOrFalseNoQuote(not netboxEnabled),
             ),
+            EnvValue(
+                os.path.join(args.configDir, 'netbox-common.env'),
+                'NETBOX_PRELOAD_PREFIXES',
+                TrueOrFalseNoQuote(netboxPreloadPrefixes),
+            ),
             # enable/disable netbox (postgres)
             EnvValue(
                 os.path.join(args.configDir, 'netbox-common.env'),
@@ -3671,6 +3680,16 @@ def main():
         default=False,
         help="Automatically populate NetBox inventory based on observed network traffic",
     )
+    netboxArgGroup.add_argument(
+        '--netbox-preload-prefixes',
+        dest='netboxPreloadPrefixes',
+        type=str2bool,
+        metavar="true|false",
+        nargs='?',
+        const=True,
+        default=False,
+        help="Preload NetBox IPAM VRFs/IP Prefixes for private IP space",
+    )
     netboxArgGroup.add_argument(
         '--netbox-site-name',
         dest='netboxSiteName',