From 308a1b0d8b595231ef8f0a31f52e8b71d017d001 Mon Sep 17 00:00:00 2001 From: SG <mero.mero.guero@gmail.com> Date: Thu, 26 Oct 2023 13:30:45 -0600 Subject: [PATCH] add option to auto-create catch-all netbox IPAM prefixes for private IP space (idaholab/Malcolm#279) --- Dockerfiles/netbox.Dockerfile | 2 + config/netbox-common.env.example | 2 + docs/kubernetes.md | 2 + docs/malcolm-hedgehog-e2e-iso-install.md | 2 + netbox/preload/prefixes_defaults.yml | 6 +++ netbox/preload/vrfs_defaults.yml | 6 +++ netbox/scripts/netbox_init.py | 56 ++++++++++++++++++------ netbox/supervisord.conf | 2 + scripts/install.py | 19 ++++++++ 9 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 netbox/preload/prefixes_defaults.yml create mode 100644 netbox/preload/vrfs_defaults.yml diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 8fbb0a628..4d5dbc7fb 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -39,6 +39,7 @@ ARG NETBOX_DEVICETYPE_LIBRARY_PATH="/opt/netbox-devicetype-library" ARG NETBOX_DEFAULT_SITE=Malcolm ARG NETBOX_CRON=true ARG NETBOX_PRELOAD_PATH="/opt/netbox-preload" +ARG NETBOX_PRELOAD_PREFIXES=false ENV NETBOX_PATH /opt/netbox ENV BASE_PATH netbox @@ -46,6 +47,7 @@ ENV NETBOX_DEVICETYPE_LIBRARY_PATH $NETBOX_DEVICETYPE_LIBRARY_PATH ENV NETBOX_DEFAULT_SITE $NETBOX_DEFAULT_SITE ENV NETBOX_CRON $NETBOX_CRON ENV NETBOX_PRELOAD_PATH $NETBOX_PRELOAD_PATH +ENV NETBOX_PRELOAD_PREFIXES $NETBOX_PRELOAD_PREFIXES ADD netbox/patch/* /tmp/netbox-patches/ diff --git a/config/netbox-common.env.example b/config/netbox-common.env.example index 882cc64ae..0caba9062 100644 --- a/config/netbox-common.env.example +++ b/config/netbox-common.env.example @@ -3,6 +3,8 @@ # The name of the default "site" to be created upon NetBox initialization, and to be queried # for enrichment (see LOGSTASH_NETBOX_ENRICHMENT) NETBOX_DEFAULT_SITE=Malcolm +# Whether or not to create catch-all VRFs/IP Prefixes for private IP space +NETBOX_PRELOAD_PREFIXES=false # Whether to disable Malcolm's NetBox instance ('true') or not ('false') NETBOX_DISABLED=true NETBOX_POSTGRES_DISABLED=true diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 3f8c9d3c5..91294d6c1 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -430,6 +430,8 @@ Should Malcolm automatically populate NetBox inventory based on observed network Specify default NetBox site name: Malcolm +Should Malcolm create "catch-all" prefixes for private IP address space? (y / N): n + Enable dark mode for OpenSearch Dashboards? (Y / n): y Malcolm has been installed to /home/user/Malcolm. See README.md for more information. diff --git a/docs/malcolm-hedgehog-e2e-iso-install.md b/docs/malcolm-hedgehog-e2e-iso-install.md index 492d8fe1f..280698e90 100644 --- a/docs/malcolm-hedgehog-e2e-iso-install.md +++ b/docs/malcolm-hedgehog-e2e-iso-install.md @@ -255,6 +255,8 @@ The [configuration and tuning](malcolm-config.md#ConfigAndTuning) wizard's quest - Answer **Y** to [populate the NetBox inventory](asset-interaction-analysis.md#NetBoxPopPassive) based on observed network traffic. Autopopulation is **not** recommended: [manual inventory population](asset-interaction-analysis.md#NetBoxPopManual) is the preferred method to create an accurate representation of the intended network design. * **Specify default NetBox site name** - NetBox has the concept of [sites](https://demo.netbox.dev/static/docs/core-functionality/sites-and-racks/); this default site name will be used as a query parameter for these enrichment lookups. +* **Should Malcolm create "catch-all" prefixes for private IP address space?** + - Answer **Y** to automatically create "catch-all" NetBox prefixes for private IP address space (i.e., one each for `10.0.0.0/8`, `172.16.0.0/12`, and `192.168.0.0/16`, respectively). This is not recommended for networks with more than one subnet. * **Should Malcolm capture live network traffic?** - Malcolm itself can perform [live analysis](live-analysis.md#LocalPCAP) of traffic it sees on another network interface (ideally not the same one used for its management). Answer **no** to this question in installations where Hedgehog Linux will be handling all network traffic capture. If users want Malcolm to observe and capture traffic instead of, or in addition to, a sensor running Hedgehog Linux, they should answer **yes** enable life traffic analysis using default settings, or select **customize** to proceed to answer the following related questions individually. - **Should Malcolm capture live network traffic to PCAP files for analysis with Arkime?** diff --git a/netbox/preload/prefixes_defaults.yml b/netbox/preload/prefixes_defaults.yml new file mode 100644 index 000000000..0e7d4c736 --- /dev/null +++ b/netbox/preload/prefixes_defaults.yml @@ -0,0 +1,6 @@ +- prefix: 10.0.0.0/8 + vrf: Private IP Space (10.0.0.0/8) +- prefix: 172.16.0.0/12 + vrf: Private IP Space (172.16.0.0/12) +- prefix: 10.0.0.0/8 + vrf: Private IP Space (192.168.0.0/16) diff --git a/netbox/preload/vrfs_defaults.yml b/netbox/preload/vrfs_defaults.yml new file mode 100644 index 000000000..866439507 --- /dev/null +++ b/netbox/preload/vrfs_defaults.yml @@ -0,0 +1,6 @@ +- enforce_unique: true + name: Private IP Space (10.0.0.0/8) +- enforce_unique: true + name: Private IP Space (172.16.0.0/12) +- enforce_unique: true + name: Private IP Space (192.168.0.0/16) diff --git a/netbox/scripts/netbox_init.py b/netbox/scripts/netbox_init.py index 0e2f500fe..a41063f9b 100755 --- a/netbox/scripts/netbox_init.py +++ b/netbox/scripts/netbox_init.py @@ -12,11 +12,14 @@ import pynetbox import randomcolor import re +import shutil import sys +import tempfile import time import malcolm_utils from collections.abc import Iterable +from distutils.dir_util import copy_tree from datetime import datetime from slugify import slugify from netbox_library_import import import_library @@ -238,6 +241,16 @@ def main(): required=False, help="Directory containing netbox-initializers files to preload", ) + parser.add_argument( + '--preload-prefixes', + dest='preloadPrefixes', + type=malcolm_utils.str2bool, + metavar="true|false", + nargs='?', + const=True, + default=malcolm_utils.str2bool(os.getenv('NETBOX_PRELOAD_PREFIXES', default='False')), + help="Preload IPAM VRFs/IP Prefixes for private IP space", + ) try: parser.error = parser.exit args = parser.parse_args() @@ -642,20 +655,35 @@ def main(): if os.path.isfile(netboxVenvPy) and os.path.isfile(manageScript) and os.path.isdir(args.preloadDir): try: with malcolm_utils.pushd(os.path.dirname(manageScript)): - retcode, output = malcolm_utils.run_process( - [ - netboxVenvPy, - os.path.basename(manageScript), - "load_initializer_data", - "--path", - args.preloadDir, - ], - logger=logging, - ) - if retcode == 0: - logging.debug(f"netbox-initializers: {retcode} {output}") - else: - logging.error(f"Error processing netbox-initializers: {retcode} {output}") + # make a local copy of the YMLs to preload + with tempfile.TemporaryDirectory() as tmpPreloadDir: + copy_tree(args.preloadDir, tmpPreloadDir) + + # only preload catch-all VRFs and IP Prefixes if explicitly specified and they don't already exist + if args.preloadPrefixes: + for loadType in ('vrfs', 'prefixes'): + defaultFileName = os.path.join(tmpPreloadDir, f'{loadType}_defaults.yml') + loadFileName = os.path.join(tmpPreloadDir, f'{loadType}.yml') + if os.path.isfile(defaultFileName) and (not os.path.isfile(loadFileName)): + try: + shutil.copyfile(defaultFileName, loadFileName) + except Exception: + pass + + retcode, output = malcolm_utils.run_process( + [ + netboxVenvPy, + os.path.basename(manageScript), + "load_initializer_data", + "--path", + tmpPreloadDir, + ], + logger=logging, + ) + if retcode == 0: + logging.debug(f"netbox-initializers: {retcode} {output}") + else: + logging.error(f"Error processing netbox-initializers: {retcode} {output}") except Exception as e: logging.error(f"{type(e).__name__} processing netbox-initializers: {e}") diff --git a/netbox/supervisord.conf b/netbox/supervisord.conf index 44e997720..8fe938f56 100644 --- a/netbox/supervisord.conf +++ b/netbox/supervisord.conf @@ -39,6 +39,8 @@ command=/opt/netbox/venv/bin/python /usr/local/bin/netbox_init.py --token "%(ENV_SUPERUSER_API_TOKEN)s" --net-map /usr/local/share/net-map.json --library "%(ENV_NETBOX_DEVICETYPE_LIBRARY_PATH)s" + --preload "%(ENV_NETBOX_PRELOAD_PATH)s" + --preload-prefixes %(ENV_NETBOX_PRELOAD_PREFIXES)s autostart=true autorestart=false startsecs=0 diff --git a/scripts/install.py b/scripts/install.py index 09285cef0..ad3b6b022 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -1263,6 +1263,10 @@ def tweak_malcolm_runtime(self, malcolm_install_path): ) if len(netboxSiteName) == 0: netboxSiteName = 'Malcolm' + netboxPreloadPrefixes = netboxEnabled and InstallerYesOrNo( + 'Should Malcolm create "catch-all" prefixes for private IP address space?', + default=args.netboxPreloadPrefixes, + ) # input packet capture parameters pcapNetSniff = False @@ -1511,6 +1515,11 @@ def tweak_malcolm_runtime(self, malcolm_install_path): 'NETBOX_DISABLED', TrueOrFalseNoQuote(not netboxEnabled), ), + EnvValue( + os.path.join(args.configDir, 'netbox-common.env'), + 'NETBOX_PRELOAD_PREFIXES', + TrueOrFalseNoQuote(netboxPreloadPrefixes), + ), # enable/disable netbox (postgres) EnvValue( os.path.join(args.configDir, 'netbox-common.env'), @@ -3671,6 +3680,16 @@ def main(): default=False, help="Automatically populate NetBox inventory based on observed network traffic", ) + netboxArgGroup.add_argument( + '--netbox-preload-prefixes', + dest='netboxPreloadPrefixes', + type=str2bool, + metavar="true|false", + nargs='?', + const=True, + default=False, + help="Preload NetBox IPAM VRFs/IP Prefixes for private IP space", + ) netboxArgGroup.add_argument( '--netbox-site-name', dest='netboxSiteName',