From ee3495ffc2ca0c3b26e8c3f4af75c8529bc54377 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Mon, 26 Aug 2024 17:05:27 -0600 Subject: [PATCH 1/7] fix bug idaholab/Malcolm#548, missing cracklib-runtime --- .../config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot | 1 - .../config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot | 1 - 2 files changed, 2 deletions(-) diff --git a/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index 528cb1b81..553c3da62 100755 --- a/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -15,7 +15,6 @@ rm -rf /var/spool/ccache # remove unwanted packages apt-get -y --purge remove \ bluez-firmware \ - cracklib-runtime \ firmware-libertas \ firmware-netronome \ firmware-nvidia* \ diff --git a/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index 196d7639a..7fba6edc2 100755 --- a/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/malcolm-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -5,7 +5,6 @@ apt-get -y --purge remove build-essential sparse $(dpkg --get-selections | grep # remove unwanted packages apt-get -y --purge remove bluez-firmware \ - cracklib-runtime \ docker-scan-plugin \ docker-buildx-plugin \ docker-ce-rootless-extras \ From b3ae65d132bbe15248a3627005a39886ff599b72 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Aug 2024 08:19:24 -0600 Subject: [PATCH 2/7] fix python watchdog package at 4.0.2 for now --- Dockerfiles/arkime.Dockerfile | 2 +- Dockerfiles/file-monitor.Dockerfile | 2 +- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/pcap-monitor.Dockerfile | 2 +- Dockerfiles/suricata.Dockerfile | 2 +- hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index d2a6413d2..a6e573a78 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -148,7 +148,7 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ mkdir -p "${ARKIME_DIR}"/plugins && \ curl -fsSL -o "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" "$(echo "${ARKIME_JA4_SO_URL}" | sed "s/XXX/${DEBARCH}/g")" && \ chmod 755 "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==4.0.2 && \ ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index fd56e1291..caa686704 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -159,7 +159,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') python-magic \ stream-zip \ supervisor \ - watchdog \ + watchdog==4.0.2 \ yara-python && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 2df341307..25272e5a3 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -101,7 +101,7 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \ unar \ unzip \ xz-utils && \ - python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog && \ + python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==4.0.2 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index 9efa4aabb..b710fcb23 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get -q update && \ python-magic \ pyzmq \ requests \ - watchdog && \ + watchdog==4.0.2 && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 50ae17df7..d2872cac8 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -108,7 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') apt-get install -q -y --no-install-recommends -t bookworm-backports \ suricata=${SURICATA_VERSION_PATTERN} \ suricata-update && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==4.0.2 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 981f3ffcb..0667b4b72 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -20,4 +20,4 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta stix2 \ stream-zip \ taxii2-client \ - watchdog + watchdog==4.0.2 From f72d1bd77b9909d7df30bed343f3b26bc5b6aa62 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Aug 2024 08:38:03 -0600 Subject: [PATCH 3/7] Reduce verbosity of pruning script --- shared/bin/prune_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/bin/prune_files.sh b/shared/bin/prune_files.sh index 29ad8d474..010327ae3 100755 --- a/shared/bin/prune_files.sh +++ b/shared/bin/prune_files.sh @@ -101,7 +101,7 @@ while true ; do if (( $DELETED_FILES > 0 )) ; then echo "Pruned $DELETED_FILES files ($(numfmt --to=iec-i --suffix=B $DELETED_BYTES)) in \"$PRUNE_PATH\"" else - echo "Nothing was pruned in \"$PRUNE_PATH\"!" >&2 + [[ "$VERBOSE" == "1" ]] && echo "Nothing was pruned in \"$PRUNE_PATH\"!" >&2 fi else From 46adb9cc64b95b3e7af176ad44491ccaa707b96d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Aug 2024 08:49:46 -0600 Subject: [PATCH 4/7] Update Watchdog package to v5.0.0 (idaholab/Malcolm#550) --- Dockerfiles/arkime.Dockerfile | 2 +- Dockerfiles/file-monitor.Dockerfile | 2 +- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/pcap-monitor.Dockerfile | 2 +- Dockerfiles/suricata.Dockerfile | 2 +- .../config/hooks/normal/0169-pip-installs.hook.chroot | 2 +- shared/bin/pcap_watcher.py | 6 +++--- shared/bin/watch_common.py | 6 +++--- shared/bin/zeek_carve_watcher.py | 6 +++--- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index a6e573a78..4b246041f 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -148,7 +148,7 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ mkdir -p "${ARKIME_DIR}"/plugins && \ curl -fsSL -o "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" "$(echo "${ARKIME_JA4_SO_URL}" | sed "s/XXX/${DEBARCH}/g")" && \ chmod 755 "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==4.0.2 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==5.0.0 && \ ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index caa686704..bfbee2de5 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -159,7 +159,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') python-magic \ stream-zip \ supervisor \ - watchdog==4.0.2 \ + watchdog==5.0.0 \ yara-python && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 25272e5a3..021077425 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -101,7 +101,7 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \ unar \ unzip \ xz-utils && \ - python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==4.0.2 && \ + python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==5.0.0 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index b710fcb23..c5187fcab 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get -q update && \ python-magic \ pyzmq \ requests \ - watchdog==4.0.2 && \ + watchdog==5.0.0 && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index d2872cac8..9ccc973a2 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -108,7 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') apt-get install -q -y --no-install-recommends -t bookworm-backports \ suricata=${SURICATA_VERSION_PATTERN} \ suricata-update && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==4.0.2 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==5.0.0 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 0667b4b72..16aba39e8 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -20,4 +20,4 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta stix2 \ stream-zip \ taxii2-client \ - watchdog==4.0.2 + watchdog==5.0.0 diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index 25816e13d..428723f95 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -46,7 +46,7 @@ from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver -from watchdog.utils import WatchdogShutdown +from watchdog.utils import WatchdogShutdownError ################################################################################################### MINIMUM_CHECKED_FILE_SIZE_DEFAULT = 24 @@ -564,9 +564,9 @@ def main(): logging.info(f"{scriptName}:\tshutting down...") if shuttingDown[0]: - raise WatchdogShutdown() + raise WatchdogShutdownError() - except WatchdogShutdown: + except WatchdogShutdownError: observer.unschedule_all() finally: diff --git a/shared/bin/watch_common.py b/shared/bin/watch_common.py index 9457a1604..e448ee8f2 100644 --- a/shared/bin/watch_common.py +++ b/shared/bin/watch_common.py @@ -20,7 +20,7 @@ from multiprocessing.pool import ThreadPool from threading import get_native_id -from watchdog.utils import WatchdogShutdown +from watchdog.utils import WatchdogShutdownError from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver from collections import namedtuple, defaultdict, OrderedDict @@ -327,9 +327,9 @@ def WatchAndProcessDirectory( observer.join(1) if shuttingDown[0]: - raise WatchdogShutdown() + raise WatchdogShutdownError() - except WatchdogShutdown: + except WatchdogShutdownError: observer.unschedule_all() finally: diff --git a/shared/bin/zeek_carve_watcher.py b/shared/bin/zeek_carve_watcher.py index 960464905..e457edd19 100755 --- a/shared/bin/zeek_carve_watcher.py +++ b/shared/bin/zeek_carve_watcher.py @@ -24,7 +24,7 @@ from multiprocessing.pool import ThreadPool from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver -from watchdog.utils import WatchdogShutdown +from watchdog.utils import WatchdogShutdownError from zeek_carve_utils import ( CAPA_VIV_MIME, @@ -320,9 +320,9 @@ def main(): logging.info(f"{scriptName}:\tshutting down...") if shuttingDown[0]: - raise WatchdogShutdown() + raise WatchdogShutdownError() - except WatchdogShutdown: + except WatchdogShutdownError: observer.unschedule_all() finally: From c652ca1ccd3b132361115c1affcc65668876fa4d Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Tue, 27 Aug 2024 08:55:54 -0600 Subject: [PATCH 5/7] Revert "Update Watchdog package to v5.0.0 (idaholab/Malcolm#550)" This reverts commit 46adb9cc64b95b3e7af176ad44491ccaa707b96d. --- Dockerfiles/arkime.Dockerfile | 2 +- Dockerfiles/file-monitor.Dockerfile | 2 +- Dockerfiles/filebeat.Dockerfile | 2 +- Dockerfiles/pcap-monitor.Dockerfile | 2 +- Dockerfiles/suricata.Dockerfile | 2 +- .../config/hooks/normal/0169-pip-installs.hook.chroot | 2 +- shared/bin/pcap_watcher.py | 6 +++--- shared/bin/watch_common.py | 6 +++--- shared/bin/zeek_carve_watcher.py | 6 +++--- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 4b246041f..a6e573a78 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -148,7 +148,7 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ mkdir -p "${ARKIME_DIR}"/plugins && \ curl -fsSL -o "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" "$(echo "${ARKIME_JA4_SO_URL}" | sed "s/XXX/${DEBARCH}/g")" && \ chmod 755 "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==5.0.0 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==4.0.2 && \ ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index bfbee2de5..caa686704 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -159,7 +159,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') python-magic \ stream-zip \ supervisor \ - watchdog==5.0.0 \ + watchdog==4.0.2 \ yara-python && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 021077425..25272e5a3 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -101,7 +101,7 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \ unar \ unzip \ xz-utils && \ - python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==5.0.0 && \ + python3 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==4.0.2 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index c5187fcab..b710fcb23 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get -q update && \ python-magic \ pyzmq \ requests \ - watchdog==5.0.0 && \ + watchdog==4.0.2 && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 9ccc973a2..d2872cac8 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -108,7 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') apt-get install -q -y --no-install-recommends -t bookworm-backports \ suricata=${SURICATA_VERSION_PATTERN} \ suricata-update && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==5.0.0 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==4.0.2 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 16aba39e8..0667b4b72 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -20,4 +20,4 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta stix2 \ stream-zip \ taxii2-client \ - watchdog==5.0.0 + watchdog==4.0.2 diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py index 428723f95..25816e13d 100755 --- a/shared/bin/pcap_watcher.py +++ b/shared/bin/pcap_watcher.py @@ -46,7 +46,7 @@ from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver -from watchdog.utils import WatchdogShutdownError +from watchdog.utils import WatchdogShutdown ################################################################################################### MINIMUM_CHECKED_FILE_SIZE_DEFAULT = 24 @@ -564,9 +564,9 @@ def main(): logging.info(f"{scriptName}:\tshutting down...") if shuttingDown[0]: - raise WatchdogShutdownError() + raise WatchdogShutdown() - except WatchdogShutdownError: + except WatchdogShutdown: observer.unschedule_all() finally: diff --git a/shared/bin/watch_common.py b/shared/bin/watch_common.py index e448ee8f2..9457a1604 100644 --- a/shared/bin/watch_common.py +++ b/shared/bin/watch_common.py @@ -20,7 +20,7 @@ from multiprocessing.pool import ThreadPool from threading import get_native_id -from watchdog.utils import WatchdogShutdownError +from watchdog.utils import WatchdogShutdown from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver from collections import namedtuple, defaultdict, OrderedDict @@ -327,9 +327,9 @@ def WatchAndProcessDirectory( observer.join(1) if shuttingDown[0]: - raise WatchdogShutdownError() + raise WatchdogShutdown() - except WatchdogShutdownError: + except WatchdogShutdown: observer.unschedule_all() finally: diff --git a/shared/bin/zeek_carve_watcher.py b/shared/bin/zeek_carve_watcher.py index e457edd19..960464905 100755 --- a/shared/bin/zeek_carve_watcher.py +++ b/shared/bin/zeek_carve_watcher.py @@ -24,7 +24,7 @@ from multiprocessing.pool import ThreadPool from watchdog.observers import Observer from watchdog.observers.polling import PollingObserver -from watchdog.utils import WatchdogShutdownError +from watchdog.utils import WatchdogShutdown from zeek_carve_utils import ( CAPA_VIV_MIME, @@ -320,9 +320,9 @@ def main(): logging.info(f"{scriptName}:\tshutting down...") if shuttingDown[0]: - raise WatchdogShutdownError() + raise WatchdogShutdown() - except WatchdogShutdownError: + except WatchdogShutdown: observer.unschedule_all() finally: From d61fdd3e1656011c1222a5d410bd1055eac64d3c Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 28 Aug 2024 07:32:44 -0600 Subject: [PATCH 6/7] idaholab/Malcolm#553, include netbox-topology plugin by default --- Dockerfiles/netbox.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 51722f9ce..e2c53ee21 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -31,6 +31,7 @@ ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$ ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV NETBOX_INITIALIZERS_VERSION "50d077d" +ENV NETBOX_TOPOLOGY_VERSION "4.0.1" ENV YQ_VERSION "4.44.3" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_" @@ -85,6 +86,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') tini && \ "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir \ "git+https://github.com/tobiasge/netbox-initializers@${NETBOX_INITIALIZERS_VERSION}" \ + "git+https://github.com/netbox-community/netbox-topology-views@v${NETBOX_TOPOLOGY_VERSION}" \ psycopg2 \ pynetbox \ python-magic \ From b4d427825e54a8dc68a75bdce0a52912a8e9dbbb Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Wed, 28 Aug 2024 07:35:05 -0600 Subject: [PATCH 7/7] Bump to v24.09.0 for development --- docker-compose-dev.yml | 46 +++++----- docker-compose.yml | 46 +++++----- docs/api-version.md | 2 +- docs/contributing-github-runners.md | 48 +++++----- docs/contributing-pcap.md | 2 +- docs/hedgehog-iso-build.md | 2 +- docs/kubernetes.md | 88 +++++++++---------- docs/malcolm-iso.md | 2 +- docs/quickstart.md | 38 ++++---- docs/third-party-envs.md | 10 +-- docs/ubuntu-install-example.md | 38 ++++---- kubernetes/03-opensearch.yml | 4 +- kubernetes/04-dashboards.yml | 2 +- kubernetes/05-upload.yml | 4 +- kubernetes/06-pcap-monitor.yml | 4 +- kubernetes/07-arkime.yml | 4 +- kubernetes/08-api.yml | 2 +- kubernetes/09-dashboards-helper.yml | 2 +- kubernetes/10-zeek.yml | 4 +- kubernetes/11-suricata.yml | 4 +- kubernetes/12-file-monitor.yml | 4 +- kubernetes/13-filebeat.yml | 4 +- kubernetes/14-logstash.yml | 4 +- kubernetes/15-netbox-redis.yml | 4 +- kubernetes/16-netbox-redis-cache.yml | 2 +- kubernetes/17-netbox-postgres.yml | 4 +- kubernetes/18-netbox.yml | 4 +- kubernetes/19-htadmin.yml | 4 +- kubernetes/20-pcap-capture.yml | 4 +- kubernetes/21-zeek-live.yml | 4 +- kubernetes/22-suricata-live.yml | 4 +- kubernetes/23-arkime-live.yml | 4 +- kubernetes/24-freq.yml | 2 +- kubernetes/98-nginx-proxy.yml | 4 +- .../aws/ami/packer_vars.json.example | 2 +- .../aws/ami/scripts/Malcolm_AMI_Setup.sh | 2 +- 36 files changed, 204 insertions(+), 204 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index a8989ea3e..1da496e08 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -5,7 +5,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:24.08.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.09.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -82,7 +82,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.08.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -135,7 +135,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:24.08.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -181,7 +181,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:24.08.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -295,7 +295,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.08.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -371,7 +371,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.08.0 + image: ghcr.io/idaholab/malcolm/arkime:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -450,7 +450,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.08.0 + image: ghcr.io/idaholab/malcolm/arkime:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -528,7 +528,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.08.0 + image: ghcr.io/idaholab/malcolm/zeek:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -600,7 +600,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.08.0 + image: ghcr.io/idaholab/malcolm/zeek:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -664,7 +664,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.08.0 + image: ghcr.io/idaholab/malcolm/suricata:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -723,7 +723,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.08.0 + image: ghcr.io/idaholab/malcolm/suricata:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -780,7 +780,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:24.08.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -835,7 +835,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:24.08.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -881,7 +881,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.08.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -935,7 +935,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:24.08.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -976,7 +976,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.08.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1028,7 +1028,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.08.0 + image: ghcr.io/idaholab/malcolm/freq:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1065,7 +1065,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.08.0 + image: ghcr.io/idaholab/malcolm/netbox:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1131,7 +1131,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.08.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1174,7 +1174,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.08.0 + image: ghcr.io/idaholab/malcolm/redis:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1221,7 +1221,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.08.0 + image: ghcr.io/idaholab/malcolm/redis:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1263,7 +1263,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.08.0 + image: ghcr.io/idaholab/malcolm/api:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1311,7 +1311,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.08.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.09.0 profiles: ["malcolm"] logging: driver: local diff --git a/docker-compose.yml b/docker-compose.yml index 705f3587a..bb5090f88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:24.08.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.09.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -76,7 +76,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.08.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -126,7 +126,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:24.08.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -169,7 +169,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:24.08.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -256,7 +256,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.08.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -329,7 +329,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:24.08.0 + image: ghcr.io/idaholab/malcolm/arkime:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -387,7 +387,7 @@ services: retries: 3 start_period: 210s arkime-live: - image: ghcr.io/idaholab/malcolm/arkime:24.08.0 + image: ghcr.io/idaholab/malcolm/arkime:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -444,7 +444,7 @@ services: source: ./pcap target: /data/pcap zeek: - image: ghcr.io/idaholab/malcolm/zeek:24.08.0 + image: ghcr.io/idaholab/malcolm/zeek:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -507,7 +507,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:24.08.0 + image: ghcr.io/idaholab/malcolm/zeek:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -562,7 +562,7 @@ services: target: /opt/zeek/share/zeek/site/custom read_only: true suricata: - image: ghcr.io/idaholab/malcolm/suricata:24.08.0 + image: ghcr.io/idaholab/malcolm/suricata:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -618,7 +618,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:24.08.0 + image: ghcr.io/idaholab/malcolm/suricata:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -672,7 +672,7 @@ services: target: /opt/suricata/include-configs read_only: true file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:24.08.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -724,7 +724,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:24.08.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -767,7 +767,7 @@ services: source: ./pcap/upload target: /pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.08.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.09.0 profiles: ["malcolm", "hedgehog"] logging: driver: local @@ -818,7 +818,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:24.08.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -859,7 +859,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.08.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -908,7 +908,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.08.0 + image: ghcr.io/idaholab/malcolm/freq:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -942,7 +942,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.08.0 + image: ghcr.io/idaholab/malcolm/netbox:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1005,7 +1005,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.08.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1045,7 +1045,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.08.0 + image: ghcr.io/idaholab/malcolm/redis:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1089,7 +1089,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.08.0 + image: ghcr.io/idaholab/malcolm/redis:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1128,7 +1128,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.08.0 + image: ghcr.io/idaholab/malcolm/api:24.09.0 profiles: ["malcolm"] logging: driver: local @@ -1170,7 +1170,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.08.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.09.0 profiles: ["malcolm"] logging: driver: local diff --git a/docs/api-version.md b/docs/api-version.md index da67ccdcf..01cc75a29 100644 --- a/docs/api-version.md +++ b/docs/api-version.md @@ -49,6 +49,6 @@ Returns version information about Malcolm and version/[health](https://opensearc } }, "sha": "dad18b1", - "version": "24.08.0" + "version": "24.09.0" } ``` diff --git a/docs/contributing-github-runners.md b/docs/contributing-github-runners.md index d009ef552..bfad5c812 100644 --- a/docs/contributing-github-runners.md +++ b/docs/contributing-github-runners.md @@ -89,7 +89,7 @@ Each container build workflow actually runs two paths in parallel: one for build ## Convenience scripts for development -As mentioned earlier, Malcolm images built using the instructions in this document are are named according to the pattern `ghcr.io/username/malcolm/image:branch`. However, note that the `image:` values found in [`docker-compose.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/docker-compose.yml) (and in the [Kubernetes](kubernetes.md#Kubernetes) [manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/)) look like `ghcr.io/idaholab/malcolm/opensearch:24.08.0`, using the OpenSearch container as an example. To run a local instance of Malcolm using these images instead of the official `ghcr.io/idaholab` ones, users will need to edit their `docker-compose.yml` file(s) and replace the `image:` tags according to this new pattern, or use the bash helper script [`./scripts/github_image_helper.sh`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/github_image_helper.sh) to pull the repository images and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.08.0`). +As mentioned earlier, Malcolm images built using the instructions in this document are are named according to the pattern `ghcr.io/username/malcolm/image:branch`. However, note that the `image:` values found in [`docker-compose.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/docker-compose.yml) (and in the [Kubernetes](kubernetes.md#Kubernetes) [manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/)) look like `ghcr.io/idaholab/malcolm/opensearch:24.09.0`, using the OpenSearch container as an example. To run a local instance of Malcolm using these images instead of the official `ghcr.io/idaholab` ones, users will need to edit their `docker-compose.yml` file(s) and replace the `image:` tags according to this new pattern, or use the bash helper script [`./scripts/github_image_helper.sh`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/github_image_helper.sh) to pull the repository images and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.09.0`). Before explaining that script, a discussion of the workflow files for the [Hedgehog Linux](live-analysis.md#Hedgehog) ([hedgehog-iso-build-docker-wrap-push-ghcr.yml ]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml)) and [Malcolm](malcolm-iso.md#ISO) ([malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -126,9 +126,9 @@ These menu options are described below: 3. GithubTriggerPackagesBuild * This option will trigger a [repository dispatch](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#repository_dispatch) via the [GitHub API](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#create-a-workflow-dispatch-event) using `curl`. In order for this operation to work, an environment variable named `GITHUB_OAUTH_TOKEN` must be defined containing a [personal access token](https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28#basic-authentication) created for your GitHub user account with the "Actions (write)" repository permission. 4. PullAndTagGithubWorkflowImages - * This option will pull latest Malcolm container images (excluding the installer ISO wrapper container images) from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.08.0`) so that they may be run without modifying the local `docker-compose.yml` file. This is probably the option users will select most often. Note that this is different from the action performed in steps 1 and 0 above: this pulls the images directly from the container registry, it does **not** extract them from the Malcolm installer ISO wrapper container image. + * This option will pull latest Malcolm container images (excluding the installer ISO wrapper container images) from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.09.0`) so that they may be run without modifying the local `docker-compose.yml` file. This is probably the option users will select most often. Note that this is different from the action performed in steps 1 and 0 above: this pulls the images directly from the container registry, it does **not** extract them from the Malcolm installer ISO wrapper container image. 5. PullAndTagGithubWorkflowISOImages - * This option will pull latest Malcolm installer ISO wrapper container images from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.08.0`). + * This option will pull latest Malcolm installer ISO wrapper container images from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.09.0`). The script can also be run non-interactively by specifying the option number on the command line (e.g., `./scripts/github_image_helper.sh 4`). @@ -167,7 +167,7 @@ $ ./scripts/github_image_helper.sh 5 PullAndTagGithubWorkflowISOImages Operation:4 PullAndTagGithubWorkflowImages -Pulling images from ghcr.io/romeogdetlevjr (main) and tagging as 24.08.0... +Pulling images from ghcr.io/romeogdetlevjr (main) and tagging as 24.09.0... main: Pulling from romeogdetlevjr/malcolm/api Digest: sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Status: Image is up to date for ghcr.io/romeogdetlevjr/malcolm/api:main @@ -181,46 +181,46 @@ ghcr.io/romeogdetlevjr/malcolm/zeek:main xxxxxxxxxxxx: Pull complete ``` -Verify that the images were pulled. Note that users will see two tags for each image: one tagged with the username and branch (e.g., `ghcr.io/romeogdetlevjr/malcolm/api:main`) and another tagged with `ghcr.io/idaholab` and the Malcolm version (e.g., `ghcr.io/idaholab/malcolm/api:24.08.0`). +Verify that the images were pulled. Note that users will see two tags for each image: one tagged with the username and branch (e.g., `ghcr.io/romeogdetlevjr/malcolm/api:main`) and another tagged with `ghcr.io/idaholab` and the Malcolm version (e.g., `ghcr.io/idaholab/malcolm/api:24.09.0`). ```bash $ docker images | grep romeogdetlevjr/malcolm -ghcr.io/idaholab/malcolm/zeek 24.08.0 xxxxxxxxxxxx 10 minutes ago 1.39GB +ghcr.io/idaholab/malcolm/zeek 24.09.0 xxxxxxxxxxxx 10 minutes ago 1.39GB ghcr.io/romeogdetlevjr/malcolm/zeek main xxxxxxxxxxxx 10 minutes ago 1.39GB -ghcr.io/idaholab/malcolm/dashboards 24.08.0 xxxxxxxxxxxx 13 minutes ago 1.55GB +ghcr.io/idaholab/malcolm/dashboards 24.09.0 xxxxxxxxxxxx 13 minutes ago 1.55GB ghcr.io/romeogdetlevjr/malcolm/dashboards main xxxxxxxxxxxx 13 minutes ago 1.55GB -ghcr.io/idaholab/malcolm/suricata 24.08.0 xxxxxxxxxxxx 14 minutes ago 339MB +ghcr.io/idaholab/malcolm/suricata 24.09.0 xxxxxxxxxxxx 14 minutes ago 339MB ghcr.io/romeogdetlevjr/malcolm/suricata main xxxxxxxxxxxx 14 minutes ago 339MB -ghcr.io/idaholab/malcolm/file-monitor 24.08.0 xxxxxxxxxxxx 15 minutes ago 712MB +ghcr.io/idaholab/malcolm/file-monitor 24.09.0 xxxxxxxxxxxx 15 minutes ago 712MB ghcr.io/romeogdetlevjr/malcolm/file-monitor main xxxxxxxxxxxx 15 minutes ago 712MB -ghcr.io/idaholab/malcolm/redis 24.08.0 xxxxxxxxxxxx 15 minutes ago 55.4MB +ghcr.io/idaholab/malcolm/redis 24.09.0 xxxxxxxxxxxx 15 minutes ago 55.4MB ghcr.io/romeogdetlevjr/malcolm/redis main xxxxxxxxxxxx 15 minutes ago 55.4MB -ghcr.io/idaholab/malcolm/nginx-proxy 24.08.0 xxxxxxxxxxxx 16 minutes ago 160MB +ghcr.io/idaholab/malcolm/nginx-proxy 24.09.0 xxxxxxxxxxxx 16 minutes ago 160MB ghcr.io/romeogdetlevjr/malcolm/nginx-proxy main xxxxxxxxxxxx 16 minutes ago 160MB -ghcr.io/idaholab/malcolm/pcap-capture 24.08.0 xxxxxxxxxxxx 16 minutes ago 137MB +ghcr.io/idaholab/malcolm/pcap-capture 24.09.0 xxxxxxxxxxxx 16 minutes ago 137MB ghcr.io/romeogdetlevjr/malcolm/pcap-capture main xxxxxxxxxxxx 16 minutes ago 137MB -ghcr.io/idaholab/malcolm/htadmin 24.08.0 xxxxxxxxxxxx 16 minutes ago 246MB +ghcr.io/idaholab/malcolm/htadmin 24.09.0 xxxxxxxxxxxx 16 minutes ago 246MB ghcr.io/romeogdetlevjr/malcolm/htadmin main xxxxxxxxxxxx 16 minutes ago 246MB ghcr.io/romeogdetlevjr/malcolm/file-upload main xxxxxxxxxxxx 16 minutes ago 250MB -ghcr.io/idaholab/malcolm/file-upload 24.08.0 xxxxxxxxxxxx 16 minutes ago 250MB -ghcr.io/idaholab/malcolm/logstash-oss 24.08.0 xxxxxxxxxxxx 16 minutes ago 1.49GB +ghcr.io/idaholab/malcolm/file-upload 24.09.0 xxxxxxxxxxxx 16 minutes ago 250MB +ghcr.io/idaholab/malcolm/logstash-oss 24.09.0 xxxxxxxxxxxx 16 minutes ago 1.49GB ghcr.io/romeogdetlevjr/malcolm/logstash-oss main xxxxxxxxxxxx 16 minutes ago 1.49GB -ghcr.io/idaholab/malcolm/netbox 24.08.0 xxxxxxxxxxxx 17 minutes ago 1.66GB +ghcr.io/idaholab/malcolm/netbox 24.09.0 xxxxxxxxxxxx 17 minutes ago 1.66GB ghcr.io/romeogdetlevjr/malcolm/netbox main xxxxxxxxxxxx 17 minutes ago 1.66GB ghcr.io/romeogdetlevjr/malcolm/filebeat-oss main xxxxxxxxxxxx 18 minutes ago 405MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.08.0 xxxxxxxxxxxx 18 minutes ago 405MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.09.0 xxxxxxxxxxxx 18 minutes ago 405MB ghcr.io/romeogdetlevjr/malcolm/postgresql main xxxxxxxxxxxx 18 minutes ago 303MB -ghcr.io/idaholab/malcolm/postgresql 24.08.0 xxxxxxxxxxxx 18 minutes ago 303MB -ghcr.io/idaholab/malcolm/arkime 24.08.0 xxxxxxxxxxxx 18 minutes ago 802MB +ghcr.io/idaholab/malcolm/postgresql 24.09.0 xxxxxxxxxxxx 18 minutes ago 303MB +ghcr.io/idaholab/malcolm/arkime 24.09.0 xxxxxxxxxxxx 18 minutes ago 802MB ghcr.io/romeogdetlevjr/malcolm/arkime main xxxxxxxxxxxx 18 minutes ago 802MB -ghcr.io/idaholab/malcolm/opensearch 24.08.0 xxxxxxxxxxxx 18 minutes ago 1.42GB +ghcr.io/idaholab/malcolm/opensearch 24.09.0 xxxxxxxxxxxx 18 minutes ago 1.42GB ghcr.io/romeogdetlevjr/malcolm/opensearch main xxxxxxxxxxxx 18 minutes ago 1.42GB -ghcr.io/idaholab/malcolm/pcap-monitor 24.08.0 xxxxxxxxxxxx 18 minutes ago 176MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.09.0 xxxxxxxxxxxx 18 minutes ago 176MB ghcr.io/romeogdetlevjr/malcolm/pcap-monitor main xxxxxxxxxxxx 18 minutes ago 176MB -ghcr.io/idaholab/malcolm/dashboards-helper 24.08.0 xxxxxxxxxxxx 18 minutes ago 233MB +ghcr.io/idaholab/malcolm/dashboards-helper 24.09.0 xxxxxxxxxxxx 18 minutes ago 233MB ghcr.io/romeogdetlevjr/malcolm/dashboards-helpermain xxxxxxxxxxxx 18 minutes ago 233MB -ghcr.io/idaholab/malcolm/freq 24.08.0 xxxxxxxxxxxx 18 minutes ago 153MB +ghcr.io/idaholab/malcolm/freq 24.09.0 xxxxxxxxxxxx 18 minutes ago 153MB ghcr.io/romeogdetlevjr/malcolm/freq main xxxxxxxxxxxx 18 minutes ago 153MB -ghcr.io/idaholab/malcolm/api 24.08.0 xxxxxxxxxxxx 18 minutes ago 169MB +ghcr.io/idaholab/malcolm/api 24.09.0 xxxxxxxxxxxx 18 minutes ago 169MB ghcr.io/romeogdetlevjr/malcolm/api main xxxxxxxxxxxx 18 minutes ago 169MB ``` diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index 21343e93d..ed874c055 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.08.0 release]({{ site.github.repository_url }}/releases/tag/v24.08.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.09.0 release]({{ site.github.repository_url }}/releases/tag/v24.09.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 6218210ec..7110efc68 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-24.08.0.iso" +Finished, created "/sensor-build/hedgehog-24.09.0.iso" … ``` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index beeeccc66..7887dbcff 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.08.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.08.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.08.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.08.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.08.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.08.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.08.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.08.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.08.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.08.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.08.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.08.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.08.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.08.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.08.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.08.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.08.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.08.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.08.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.08.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.08.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.08.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.09.0 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.09.0 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.09.0 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.09.0 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.09.0 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.09.0 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.09.0 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.09.0 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.09.0 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.09.0 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.09.0 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.09.0 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.09.0 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.09.0 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.09.0 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.09.0 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.09.0 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.09.0 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.09.0 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.09.0 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.09.0 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.09.0 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -554,28 +554,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.08.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.08.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.08.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.08.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.08.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.08.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.08.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.08.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.08.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.08.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.08.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.08.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.08.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.08.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.08.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.08.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.08.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.08.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.08.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.08.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.08.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.08.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.09.0 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.09.0 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.09.0 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.09.0 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.09.0 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.09.0 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.09.0 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.09.0 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.09.0 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.09.0 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.09.0 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.09.0 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.09.0 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.09.0 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.09.0 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.09.0 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.09.0 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.09.0 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.09.0 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.09.0 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.09.0 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.09.0 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index e93d2a0eb..a8414b298 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-24.08.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-24.09.0.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index 906272f21..94b05e971 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.08.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.08.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.08.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.08.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.08.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.08.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.08.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.08.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.08.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.08.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.08.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.08.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.08.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.08.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.08.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.08.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.08.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.08.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.08.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.09.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.09.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.09.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.09.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.09.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.09.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.09.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.09.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.09.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.09.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.09.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.09.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.09.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.09.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.09.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.09.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.09.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.09.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.09.0 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/third-party-envs.md b/docs/third-party-envs.md index 4382fd76b..f62b2ccbc 100644 --- a/docs/third-party-envs.md +++ b/docs/third-party-envs.md @@ -43,7 +43,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro amazon-ebs: output will be in this color. ==> amazon-ebs: Prevalidating any provided VPC information - ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.08.0-arm64-2024-05-30T13-57-31Z + ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.09.0-arm64-2024-05-30T13-57-31Z amazon-ebs: Found Image ID: ami-xxxxxxxxxxxxxxxxx ... @@ -53,7 +53,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro ==> amazon-ebs: Adding tags to AMI (ami-xxxxxxxxxxxxxxxxx)... ==> amazon-ebs: Tagging snapshot: snap-xxxxxxxxxxxxxxxxx ==> amazon-ebs: Creating AMI tags - amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.08.0" + amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.09.0" amazon-ebs: Adding tag: "source_ami_name": "amzn2-ami-kernel-5.10-hvm-2.0.20240521.0-arm64-gp2" ==> amazon-ebs: Creating snapshot tags ==> amazon-ebs: Terminating the source AWS instance... @@ -79,7 +79,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro "Architecture": "arm64", "CreationDate": "2024-05-30T14:02:21.000Z", "ImageId": "ami-xxxxxxxxxxxxxxxxx", - "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.08.0-arm64-2024-05-30T13-57-31Z", + "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.09.0-arm64-2024-05-30T13-57-31Z", "ImageType": "machine", "Public": false, "OwnerId": "xxxxxxxxxxxx", @@ -100,14 +100,14 @@ The files referenced in this section can be found in [scripts/third-party-enviro ], "EnaSupport": true, "Hypervisor": "xen", - "Name": "malcolm-v24.08.0-arm64-2024-05-30T13-57-31Z", + "Name": "malcolm-v24.09.0-arm64-2024-05-30T13-57-31Z", "RootDeviceName": "/dev/xvda", "RootDeviceType": "ebs", "SriovNetSupport": "simple", "Tags": [ { "Key": "Malcolm", - "Value": "idaholab/Malcolm/v24.08.0" + "Value": "idaholab/Malcolm/v24.09.0" }, { "Key": "source_ami_name", diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 26fa5ba09..960ef6884 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -257,25 +257,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.08.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.08.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.08.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.08.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.08.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.08.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.08.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.08.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.08.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.08.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.08.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.08.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.08.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.08.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.08.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.08.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.08.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.08.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.08.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.09.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.09.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.09.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.09.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.09.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.09.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.09.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.09.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.09.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.09.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.09.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.09.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.09.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.09.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.09.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.09.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.09.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.09.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.09.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 680e009db..73757072e 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:24.08.0 + image: ghcr.io/idaholab/malcolm/opensearch:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 4652718b8..6db70b00f 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:24.08.0 + image: ghcr.io/idaholab/malcolm/dashboards:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index f13854396..c32afadc6 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:24.08.0 + image: ghcr.io/idaholab/malcolm/file-upload:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 8525b6121..7450ac064 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.08.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 69ca02bf0..eb5048e9f 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:24.08.0 + image: ghcr.io/idaholab/malcolm/arkime:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -79,7 +79,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index c4cec03dc..3d57ac4ad 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:24.08.0 + image: ghcr.io/idaholab/malcolm/api:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index d755c8b2e..7ad4015c5 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.08.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index de874a2ae..9c244f867 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:24.08.0 + image: ghcr.io/idaholab/malcolm/zeek:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index bff7d6315..ca41760c9 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:24.08.0 + image: ghcr.io/idaholab/malcolm/suricata:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index 7dd841c2d..0b17bc114 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:24.08.0 + image: ghcr.io/idaholab/malcolm/file-monitor:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index 85b28f9f5..02b8e9d2a 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.08.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index fe980e8e8..955626096 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:24.08.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -115,7 +115,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 952a9bd15..450ec2fad 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:24.08.0 + image: ghcr.io/idaholab/malcolm/redis:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 1e0c54472..2f419e85c 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:24.08.0 + image: ghcr.io/idaholab/malcolm/redis:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index c7c65353f..6167c1c92 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:24.08.0 + image: ghcr.io/idaholab/malcolm/postgresql:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index f8ad3dfdf..11d1d8d93 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:24.08.0 + image: ghcr.io/idaholab/malcolm/netbox:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index 1ebabcec4..d592c32bc 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:24.08.0 + image: ghcr.io/idaholab/malcolm/htadmin:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index e23e69de4..130570b12 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:24.08.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 7174ff5b8..59cab5575 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:24.08.0 + image: ghcr.io/idaholab/malcolm/zeek:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index ee5512dc1..aefa5b1de 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:24.08.0 + image: ghcr.io/idaholab/malcolm/suricata:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index 57c22ba87..5684d4d57 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/idaholab/malcolm/arkime:24.08.0 + image: ghcr.io/idaholab/malcolm/arkime:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index ee8175e2a..8792acf61 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:24.08.0 + image: ghcr.io/idaholab/malcolm/freq:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index 1fb13cfff..31e1ca3d0 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.08.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.09.0 imagePullPolicy: Always stdin: false tty: true @@ -99,7 +99,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.08.0 + image: ghcr.io/idaholab/malcolm/dirinit:24.09.0 imagePullPolicy: Always stdin: false tty: true diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 8c853828f..04cdb45e0 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -3,7 +3,7 @@ "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", "instance_arch": "x86_64", - "malcolm_tag": "v24.08.0", + "malcolm_tag": "v24.09.0", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh index 9c91840eb..4b2c6bb71 100755 --- a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh +++ b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh @@ -32,7 +32,7 @@ fi # -u UID (user UID, e.g., 1000) VERBOSE_FLAG= MALCOLM_REPO=${MALCOLM_REPO:-idaholab/Malcolm} -MALCOLM_TAG=${MALCOLM_TAG:-v24.08.0} +MALCOLM_TAG=${MALCOLM_TAG:-v24.09.0} [[ -z "$MALCOLM_UID" ]] && ( [[ $EUID -eq 0 ]] && MALCOLM_UID=1000 || MALCOLM_UID="$(id -u)" ) while getopts 'vr:t:u:' OPTION; do case "$OPTION" in