diff --git a/.github/workflows/api-build-and-push-ghcr.yml b/.github/workflows/api-build-and-push-ghcr.yml index 939007b3b..5cfd3ead3 100644 --- a/.github/workflows/api-build-and-push-ghcr.yml +++ b/.github/workflows/api-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/arkime-build-and-push-ghcr.yml b/.github/workflows/arkime-build-and-push-ghcr.yml index 32016cab4..2effb3957 100644 --- a/.github/workflows/arkime-build-and-push-ghcr.yml +++ b/.github/workflows/arkime-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/dashboards-build-and-push-ghcr.yml b/.github/workflows/dashboards-build-and-push-ghcr.yml index 1452ed146..1c0ab778c 100644 --- a/.github/workflows/dashboards-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml index 7f8bf5804..fd862543f 100644 --- a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml +++ b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/file-monitor-build-and-push-ghcr.yml b/.github/workflows/file-monitor-build-and-push-ghcr.yml index bafd62550..fbb6bbbfd 100644 --- a/.github/workflows/file-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/file-monitor-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*.sh' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/file-upload-build-and-push-ghcr.yml b/.github/workflows/file-upload-build-and-push-ghcr.yml index f1a5b3113..b49ae4bea 100644 --- a/.github/workflows/file-upload-build-and-push-ghcr.yml +++ b/.github/workflows/file-upload-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/filebeat-build-and-push-ghcr.yml b/.github/workflows/filebeat-build-and-push-ghcr.yml index e52a73691..10be9650e 100644 --- a/.github/workflows/filebeat-build-and-push-ghcr.yml +++ b/.github/workflows/filebeat-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/freq-build-and-push-ghcr.yml b/.github/workflows/freq-build-and-push-ghcr.yml index 84a3254e0..d27e67429 100644 --- a/.github/workflows/freq-build-and-push-ghcr.yml +++ b/.github/workflows/freq-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/htadmin-build-and-push-ghcr.yml b/.github/workflows/htadmin-build-and-push-ghcr.yml index eff564dee..06fbcabac 100644 --- a/.github/workflows/htadmin-build-and-push-ghcr.yml +++ b/.github/workflows/htadmin-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/logstash-build-and-push-ghcr.yml b/.github/workflows/logstash-build-and-push-ghcr.yml index 6e8ed88cd..997d18d2f 100644 --- a/.github/workflows/logstash-build-and-push-ghcr.yml +++ b/.github/workflows/logstash-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml index fc9a44507..80aef51ce 100644 --- a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml +++ b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -9,6 +9,7 @@ on: - 'malcolm-iso/**' - 'shared/bin/*' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_iso_workflow_build' - '.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml' workflow_dispatch: diff --git a/.github/workflows/netbox-build-and-push-ghcr.yml b/.github/workflows/netbox-build-and-push-ghcr.yml index a8ecbc443..2693f9323 100644 --- a/.github/workflows/netbox-build-and-push-ghcr.yml +++ b/.github/workflows/netbox-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/nginx-build-and-push-ghcr.yml b/.github/workflows/nginx-build-and-push-ghcr.yml index 4489ce37a..b89a4d859 100644 --- a/.github/workflows/nginx-build-and-push-ghcr.yml +++ b/.github/workflows/nginx-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' - '_config.yml' - '_includes/**' diff --git a/.github/workflows/opensearch-build-and-push-ghcr.yml b/.github/workflows/opensearch-build-and-push-ghcr.yml index c6faee5d6..08f5967fd 100644 --- a/.github/workflows/opensearch-build-and-push-ghcr.yml +++ b/.github/workflows/opensearch-build-and-push-ghcr.yml @@ -13,6 +13,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/pcap-capture-build-and-push-ghcr.yml b/.github/workflows/pcap-capture-build-and-push-ghcr.yml index 057d4cfc9..f3a224290 100644 --- a/.github/workflows/pcap-capture-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-capture-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml index 6a69b2bad..eab99d9d1 100644 --- a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml +++ b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/postgresql-build-and-push-ghcr.yml b/.github/workflows/postgresql-build-and-push-ghcr.yml index 703730e6d..e916b4360 100644 --- a/.github/workflows/postgresql-build-and-push-ghcr.yml +++ b/.github/workflows/postgresql-build-and-push-ghcr.yml @@ -13,6 +13,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/redis-build-and-push-ghcr.yml b/.github/workflows/redis-build-and-push-ghcr.yml index ed496a575..c36708dcf 100644 --- a/.github/workflows/redis-build-and-push-ghcr.yml +++ b/.github/workflows/redis-build-and-push-ghcr.yml @@ -13,6 +13,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/.github/workflows/suricata-build-and-push-ghcr.yml b/.github/workflows/suricata-build-and-push-ghcr.yml index 764a2737f..73af7da1b 100644 --- a/.github/workflows/suricata-build-and-push-ghcr.yml +++ b/.github/workflows/suricata-build-and-push-ghcr.yml @@ -14,6 +14,7 @@ on: - '!shared/bin/sensor-init.sh' - '!shared/bin/configure-interfaces.py' - '!shared/bin/configure-capture.py' + - '!shared/bin/zeek*' - '.trigger_workflow_build' workflow_dispatch: repository_dispatch: diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 99a1fd88f..cc4a1b851 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -4,7 +4,7 @@ FROM debian:11-slim AS build ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "v4.3.0" +ENV ARKIME_VERSION "v4.3.1" ENV ARKIME_DIR "/opt/arkime" ENV ARKIME_URL "https://github.com/arkime/arkime.git" ENV ARKIME_LOCALELASTICSEARCH no diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index c6ccfcb93..cdbc9850e 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -14,10 +14,10 @@ ENV PGROUP "dashboarder" ENV TERM xterm -ARG OPENSEARCH_VERSION="2.6.0" +ARG OPENSEARCH_VERSION="2.7.0" ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION -ARG OPENSEARCH_DASHBOARDS_VERSION="2.6.0" +ARG OPENSEARCH_DASHBOARDS_VERSION="2.7.0" ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION # base system dependencies for checking out and building plugins @@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \ # runtime ################################################################## -FROM opensearchproject/opensearch-dashboards:2.6.0 +FROM opensearchproject/opensearch-dashboards:2.7.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -90,7 +90,7 @@ ENV PUSER_PRIV_DROP true ENV TERM xterm ENV TINI_VERSION v0.19.0 -ENV OSD_TRANSFORM_VIS_VERSION 2.6.0 +ENV OSD_TRANSFORM_VIS_VERSION 2.7.0 ARG OPENSEARCH_URL="http://opensearch:9200" ARG OPENSEARCH_LOCAL="true" @@ -114,6 +114,7 @@ USER root COPY --from=build /usr/share/opensearch-dashboards/plugins/sankey_vis/build/kbnSankeyVis.zip /tmp/kbnSankeyVis.zip ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini +ADD https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip /tmp/transformVis.zip RUN yum upgrade -y && \ yum install -y curl psmisc util-linux openssl rsync python3 zip unzip && \ @@ -122,7 +123,14 @@ RUN yum upgrade -y && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \ - /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \ + cd /tmp && \ + # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + # sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + # sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/package.json && \ + # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + cd /usr/share/opensearch-dashboards/plugins && \ + /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ + rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \ chown --silent -R root:root /usr/share/opensearch-dashboards/plugins/* \ /usr/share/opensearch-dashboards/node_modules/* \ /usr/share/opensearch-dashboards/src/* && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index f3bcd19d0..799cd6a44 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.6.2 +FROM docker.elastic.co/beats/filebeat-oss:8.7.1 # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index a546542aa..b1beda6e7 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -73,7 +73,8 @@ RUN apt-get -q update && \ mv /etc/unit/nginx-unit-new.json /etc/unit/nginx-unit.json && \ chmod 644 /etc/unit/nginx-unit.json && \ tr -cd '\11\12\15\40-\176' < /opt/netbox/netbox/netbox/configuration.py > /opt/netbox/netbox/netbox/configuration_ascii.py && \ - mv /opt/netbox/netbox/netbox/configuration_ascii.py /opt/netbox/netbox/netbox/configuration.py + mv /opt/netbox/netbox/netbox/configuration_ascii.py /opt/netbox/netbox/netbox/configuration.py && \ + sed -i -E 's@^([[:space:]]*\-\-(state|tmp))([[:space:]])@\1dir\3@g' /opt/netbox/launch-netbox.sh COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/ COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/ diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index a65b0a1c5..aeb6a6cee 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch:2.6.0 +FROM opensearchproject/opensearch:2.7.0 # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/config/logstash.env.example b/config/logstash.env.example index f8aa43202..2e640b86a 100644 --- a/config/logstash.env.example +++ b/config/logstash.env.example @@ -12,4 +12,4 @@ LOGSTASH_REVERSE_DNS=false # Whether or not Logstash will enrich network traffic metadata via NetBox API calls LOGSTASH_NETBOX_ENRICHMENT=false # Logstash memory allowance and other Java options -LS_JAVA_OPTS=-server -Xms2g -Xmx2g -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file +LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file diff --git a/config/opensearch.env.example b/config/opensearch.env.example index 258e23233..9503d9078 100644 --- a/config/opensearch.env.example +++ b/config/opensearch.env.example @@ -37,7 +37,7 @@ OPENSEARCH_SECONDARY_CREDS_CONFIG_FILE=/var/local/curlrc/.opensearch.secondary.c # certificates). OPENSEARCH_SECONDARY_SSL_CERTIFICATE_VERIFICATION=false # OpenSearch memory allowance and other Java options -OPENSEARCH_JAVA_OPTS=-server -Xms4g -Xmx4g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true +OPENSEARCH_JAVA_OPTS=-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true logger.level=WARN bootstrap.memory_lock=true diff --git a/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json b/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json index 113bae5ca..4e214d244 100644 --- a/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json +++ b/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json @@ -12,7 +12,7 @@ "attributes": { "title": "GENISYS", "hits": 0, - "description": "Dashboard for the DNP3 Protocol", + "description": "Dashboard for the GENISYS Protocol", "panelsJSON": "[{\"version\":\"1.3.1\",\"gridData\":{\"h\":28,\"i\":\"1\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":10,\"i\":\"58856fb7-efd0-4246-9dc9-d8b0d5c3fcba\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"58856fb7-efd0-4246-9dc9-d8b0d5c3fcba\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":10,\"i\":\"c078d6a7-456e-4fed-80c6-f36123c3ba82\",\"w\":32,\"x\":16,\"y\":0},\"panelIndex\":\"c078d6a7-456e-4fed-80c6-f36123c3ba82\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"c04b22a5-6b7e-4c18-8172-d39ec8549e4a\",\"w\":8,\"x\":8,\"y\":10},\"panelIndex\":\"c04b22a5-6b7e-4c18-8172-d39ec8549e4a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"4da40cc7-ad85-4dd1-88cf-8b207995c932\",\"w\":12,\"x\":16,\"y\":10},\"panelIndex\":\"4da40cc7-ad85-4dd1-88cf-8b207995c932\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_4\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"74347ef4-7a00-4d8f-a172-120339fd5e30\",\"w\":20,\"x\":28,\"y\":10},\"panelIndex\":\"74347ef4-7a00-4d8f-a172-120339fd5e30\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"40ffbd38-1edc-4493-b313-6f65729cbe70\",\"w\":16,\"x\":0,\"y\":28},\"panelIndex\":\"40ffbd38-1edc-4493-b313-6f65729cbe70\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"panelRefName\":\"panel_6\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"2cb13858-f268-4cd4-8207-3932c70dc83a\",\"w\":12,\"x\":16,\"y\":28},\"panelIndex\":\"2cb13858-f268-4cd4-8207-3932c70dc83a\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}},\"table\":null},\"panelRefName\":\"panel_7\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"7aabaf8b-4a54-48df-ac8e-c732327f420e\",\"w\":20,\"x\":28,\"y\":28},\"panelIndex\":\"7aabaf8b-4a54-48df-ac8e-c732327f420e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":31,\"i\":\"6b987e44-72f1-4e33-9fa3-cb21c7313829\",\"w\":48,\"x\":0,\"y\":46},\"panelIndex\":\"6b987e44-72f1-4e33-9fa3-cb21c7313829\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"}]", "optionsJSON": "{\"useMargins\":true}", "version": 1, diff --git a/dashboards/templates/composable/component/arkime.json b/dashboards/templates/composable/component/arkime.json index b1b975fd5..678585380 100644 --- a/dashboards/templates/composable/component/arkime.json +++ b/dashboards/templates/composable/component/arkime.json @@ -11,6 +11,10 @@ "destination.geo.longitude": { "type": "float" }, "dns.host": { "type": "keyword" }, "firstPacket": { "type": "date" }, + "http.xffASN": { "type": "keyword" }, + "http.xffGEO": { "type": "keyword" }, + "http.xffIp": { "type": "ip" }, + "http.xffRIR": { "type": "keyword" }, "lastPacket": { "type": "date" }, "node": { "type": "keyword" }, "protocol": { "type": "keyword" }, diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index f1efc89c0..f72ca6903 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -4,7 +4,7 @@ version: '3.7' services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:23.05.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.05.1 restart: "no" stdin_open: false tty: true @@ -37,7 +37,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.1 restart: "no" stdin_open: false tty: true @@ -64,7 +64,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.05.1 restart: "no" stdin_open: false tty: true @@ -90,7 +90,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.1 restart: "no" stdin_open: false tty: true @@ -132,7 +132,7 @@ services: retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.1 restart: "no" stdin_open: false tty: true @@ -167,7 +167,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:23.05.0 + image: ghcr.io/idaholab/malcolm/arkime:23.05.1 restart: "no" stdin_open: false tty: true @@ -203,7 +203,7 @@ services: retries: 3 start_period: 210s zeek: - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 restart: "no" stdin_open: false tty: true @@ -241,7 +241,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 restart: "no" stdin_open: false tty: true @@ -269,7 +269,7 @@ services: - ./zeek-logs/extract_files:/zeek/extract_files - ./zeek/intel:/opt/zeek/share/zeek/site/intel suricata: - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 restart: "no" stdin_open: false tty: true @@ -305,7 +305,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 restart: "no" stdin_open: false tty: true @@ -331,7 +331,7 @@ services: - ./suricata-logs:/var/log/suricata - ./suricata/rules:/opt/suricata/rules:ro file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.05.1 restart: "no" stdin_open: false tty: true @@ -357,7 +357,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.1 restart: "no" stdin_open: false tty: true @@ -379,7 +379,7 @@ services: - ./nginx/ca-trust:/var/local/ca-trust:ro - ./pcap/upload:/pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.1 restart: "no" stdin_open: false tty: true @@ -405,7 +405,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:23.05.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.05.1 restart: "no" stdin_open: false tty: true @@ -433,7 +433,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:23.05.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.05.1 restart: "no" stdin_open: false tty: true @@ -458,7 +458,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:23.05.0 + image: ghcr.io/idaholab/malcolm/freq:23.05.1 restart: "no" stdin_open: false tty: true @@ -480,7 +480,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:23.05.0 + image: ghcr.io/idaholab/malcolm/netbox:23.05.1 restart: "no" stdin_open: false tty: true @@ -513,7 +513,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:23.05.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.05.1 restart: "no" stdin_open: false tty: true @@ -537,7 +537,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 restart: "no" stdin_open: false tty: true @@ -565,7 +565,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 restart: "no" stdin_open: false tty: true @@ -592,7 +592,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:23.05.0 + image: ghcr.io/idaholab/malcolm/api:23.05.1 command: gunicorn --bind 0:5000 manage:app restart: "no" stdin_open: false @@ -616,7 +616,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.1 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index e2a7bbed1..1acbbe5f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:23.05.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.05.1 restart: "no" stdin_open: false tty: true @@ -43,7 +43,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.1 restart: "no" stdin_open: false tty: true @@ -73,7 +73,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.05.1 restart: "no" stdin_open: false tty: true @@ -102,7 +102,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.1 restart: "no" stdin_open: false tty: true @@ -151,7 +151,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.1 restart: "no" stdin_open: false tty: true @@ -189,7 +189,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:23.05.0 + image: ghcr.io/idaholab/malcolm/arkime:23.05.1 restart: "no" stdin_open: false tty: true @@ -231,7 +231,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 restart: "no" stdin_open: false tty: true @@ -273,7 +273,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 restart: "no" stdin_open: false tty: true @@ -305,7 +305,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 restart: "no" stdin_open: false tty: true @@ -344,7 +344,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 restart: "no" stdin_open: false tty: true @@ -373,7 +373,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.05.1 restart: "no" stdin_open: false tty: true @@ -402,7 +402,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.1 restart: "no" stdin_open: false tty: true @@ -427,7 +427,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.1 restart: "no" stdin_open: false tty: true @@ -456,7 +456,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:23.05.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.05.1 restart: "no" stdin_open: false tty: true @@ -484,7 +484,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:23.05.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.05.1 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -512,7 +512,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:23.05.0 + image: ghcr.io/idaholab/malcolm/freq:23.05.1 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -537,7 +537,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:23.05.0 + image: ghcr.io/idaholab/malcolm/netbox:23.05.1 build: context: . dockerfile: Dockerfiles/netbox.Dockerfile @@ -574,7 +574,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:23.05.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.05.1 build: context: . dockerfile: Dockerfiles/postgresql.Dockerfile @@ -601,7 +601,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 build: context: . dockerfile: Dockerfiles/redis.Dockerfile @@ -632,7 +632,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 build: context: . dockerfile: Dockerfiles/redis.Dockerfile @@ -662,7 +662,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:23.05.0 + image: ghcr.io/idaholab/malcolm/api:23.05.1 build: context: . dockerfile: Dockerfiles/api.Dockerfile @@ -692,7 +692,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.1 restart: "no" stdin_open: false tty: true diff --git a/docs/README.md b/docs/README.md index cdcc3f161..1e9e9d0d2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -98,6 +98,11 @@ For smaller networks, use at home by network security enthusiasts, or in the fie - [Setup](malcolm-iso.md#ISOSetup) - [Time synchronization](time-sync.md#ConfigTime) * [Deploying Malcolm with Kubernetes](kubernetes.md#Kubernetes) + - [Configuration](kubernetes.md#Config) + - [Running Malcolm](kubernetes.md#Running) + - [Deployment Example](kubernetes.md#Example) + - [Future Enhancements](kubernetes.md#Future) + - [Deploying Malcolm on Amazon Elastic Kubernetes Service (EKS)](kubernetes-eks.md#KubernetesEKS) * [Hardening](hardening.md#Hardening) - [Compliance Exceptions](hardening.md#ComplianceExceptions) * [Installation example using Ubuntu 22.04 LTS](ubuntu-install-example.md#InstallationExample) diff --git a/docs/download.md b/docs/download.md index a9c6b154f..da4b31a3f 100644 --- a/docs/download.md +++ b/docs/download.md @@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [malcolm-23.05.0.iso](/iso/malcolm-23.05.0.iso) (5.3GiB) | [`e9e00694f25b9d0dcc286496490e184930611ddbed6c52dfab77a935d2afa850`](/iso/malcolm-23.05.0.iso.sha256.txt) | +| [malcolm-23.05.1.iso](/iso/malcolm-23.05.1.iso) (5.4GiB) | [`03e3d3cc9fbd334c04c6eef7e83debea203503fe3f5dba665ebb654c26056792`](/iso/malcolm-23.05.1.iso.sha256.txt) | ## Hedgehog Linux @@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno | ISO | SHA256 | |---|---| -| [hedgehog-23.05.0.iso](/iso/hedgehog-23.05.0.iso) (2.3GiB) | [`f850ecd3b62731b46ac0366bdcdd62437da30220c23f94013873c6c92cbddff7`](/iso/hedgehog-23.05.0.iso.sha256.txt) | +| [hedgehog-23.05.1.iso](/iso/hedgehog-23.05.1.iso) (2.3GiB) | [`ad14e0e51cf51966a3c54b117e668ff588fc6a94fb5a5147c373d6c5b3b3990d`](/iso/hedgehog-23.05.1.iso.sha256.txt) | ## Warning diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index dcf35e219..e13bb5ffb 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-23.05.0.iso" +Finished, created "/sensor-build/hedgehog-23.05.1.iso" … ``` diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md new file mode 100644 index 000000000..62f3b1293 --- /dev/null +++ b/docs/kubernetes-eks.md @@ -0,0 +1,350 @@ +# Deploying Malcolm on Amazon Elastic Kubernetes Service (EKS) + +This document outlines the process of setting up a cluster on [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/) using [Amazon Web Services](https://aws.amazon.com/) in preparation for [**Deploying Malcolm with Kubernetes**](kubernetes.md). + +This is a work-in-progress document that is still a bit rough around the edges. You'll need to replace things like `cluster-name` and `us-east-1` with the values that are appliable to your cluster. Any feedback is welcome in the [relevant issue](https://github.com/idaholab/Malcolm/issues/194) on GitHub. + +## Prerequisites + +* [aws cli](https://aws.amazon.com/cli/) with functioning access to your AWS infrastructure +* [eksctl](https://eksctl.io/) + +## Procedure + +1. Create a [VPC](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#vpcs:) with subnets in 2 or more availability zones +1. Create a [security group](https://us-east-1.console.aws.amazon.com/vpc/home?region=us-east-1#SecurityGroups:) for VPC +1. Create an [EKS cluster](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters) +1. Generate a kubeconfig file to use with Malcolm's control scripts (`malcolmeks.yaml` is used in this example) + ```bash + aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml + ``` +1. Create a [node group](https://us-east-1.console.aws.amazon.com/eks/home?region=us-east-1#/clusters/cluster-name/add-node-group) +1. [Deploy](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) `metrics-server` + ```bash + kubectl --kubeconfig=malcolmeks.yaml apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + ``` +1. Deploy ingress-nginx as described [here](kubernetes.md#Ingress). [This script (`deploy_ingress_nginx.sh`)]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) may be helpful in doing so. To [provide external access](https://repost.aws/knowledge-center/eks-access-kubernetes-services) to services in the EKS cluster, pass `-a -e` to `deploy_ingress_nginx.sh` +1. Associate IAM OIDC provider with cluster + ```bash + eksctl utils associate-iam-oidc-provider --region=us-east-1 --cluster=cluster-name --approve + ``` +1. [deploy Amazon EFS CSI driver](https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html) + * review **Prerequisites** + * follow steps for **Create an IAM policy and role** + * follow steps for **Install the Amazon EFS driver** + * follow steps for **Create an Amazon [EFS file system](https://docs.aws.amazon.com/efs/latest/ug/gs-step-two-create-efs-resources.html)** +1. [Create and launch an EC2 instance](https://docs.aws.amazon.com/efs/latest/ug/gs-step-one-create-ec2-resources.html) for initializing the directory structure on the EFS filesystem (this can be a very small instance, e.g., t2.micro). Make sure when configuring this instance you give configure to the EFS file system in the storage configuration. +1. SSH to instance and initialize NFS subdirectories + - set up malcolm subdirectory + ```bash + sudo touch /mnt/efs/fs1/test-file.txt + sudo mkdir -p /mnt/efs/fs1/malcolm + sudo chown 1000:1000 /mnt/efs/fs1/malcolm + ``` + - `/mnt/efs/fs1/malcolm/init_storage.sh` + ```bash + #!/bin/bash + + if [ -z "$BASH_VERSION" ]; then + echo "Wrong interpreter, please run \"$0\" with bash" + exit 1 + fi + + ENCODING="utf-8" + + RUN_PATH="$(pwd)" + [[ "$(uname -s)" = 'Darwin' ]] && REALPATH=grealpath || REALPATH=realpath + [[ "$(uname -s)" = 'Darwin' ]] && DIRNAME=gdirname || DIRNAME=dirname + if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then + echo "$(basename "${BASH_SOURCE[0]}") requires $REALPATH and $DIRNAME" + exit 1 + fi + SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" + pushd "$SCRIPT_PATH" >/dev/null 2>&1 + + rm -rf ./opensearch/* ./opensearch-backup/* ./pcap/* ./suricata-logs/* ./zeek-logs/* ./config/netbox/* ./config/zeek/* ./runtime-logs/* + mkdir -vp ./config/auth ./config/htadmin ./config/opensearch ./config/logstash ./config/netbox/media ./config/netbox/postgres ./config/netbox/redis ./config/zeek/intel/MISP ./config/zeek/intel/STIX ./opensearch ./opensearch-backup ./pcap/upload ./pcap/processed ./suricata-logs ./zeek-logs/current ./zeek-logs/upload ./zeek-logs/extract_files ./runtime-logs/arkime ./runtime-logs/nginx + + popd >/dev/null 2>&1 + ``` + ```bash + /mnt/efs/fs1/malcolm/init_storage.sh + mkdir: created directory './config/netbox/media' + mkdir: created directory './config/netbox/postgres' + mkdir: created directory './config/netbox/redis' + mkdir: created directory './config/zeek/intel' + mkdir: created directory './config/zeek/intel/MISP' + mkdir: created directory './config/zeek/intel/STIX' + mkdir: created directory './pcap/upload' + mkdir: created directory './pcap/processed' + mkdir: created directory './zeek-logs/current' + mkdir: created directory './zeek-logs/upload' + mkdir: created directory './zeek-logs/extract_files' + mkdir: created directory './runtime-logs' + ``` +1. Set up [access points](https://docs.aws.amazon.com/efs/latest/ug/efs-access-points.html), and note the **Access point ID**s to put in your YAML in the next step + + | name | mountpoint | access point ID | + | ----------------- | -------------------------- | ---------------------- | + | config | /malcolm/config | fsap-config | + | opensearch | /malcolm/opensearch | fsap-opensearch | + | opensearch-backup | /malcolm/opensearch-backup | fsap-opensearch-backup | + | pcap | /malcolm/pcap | fsap-pcap | + | runtime-logs | /malcolm/runtime-logs | fsap-runtime-logs | + | suricata-logs | /malcolm/suricata-logs | fsap-suricata-logs | + | zeek-logs | /malcolm/zeek-logs | fsap-zeek-logs | + +1. Create YAML for persistent volumes and volume claims from the EBS Volume ID. In this example, replace `fs-FILESYSTEMID` with your EFS filesystem ID and `fsap-XXXXXXXX` with the appropriate access point ID + ```yaml + apiVersion: v1 + kind: PersistentVolume + metadata: + name: pcap-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-pcap + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: pcap-claim + namespace: malcolm + spec: + storageClassName: efs-sc + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 500Gi + volumeName: pcap-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: zeek-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 250Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-zeek-logs + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: zeek-claim + namespace: malcolm + spec: + storageClassName: efs-sc + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 250Gi + volumeName: zeek-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: suricata-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 100Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-suricata-logs + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: suricata-claim + namespace: malcolm + spec: + storageClassName: efs-sc + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 100Gi + volumeName: suricata-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: config-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 25Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-config + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: config-claim + namespace: malcolm + spec: + storageClassName: efs-sc + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 25Gi + volumeName: config-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: runtime-logs-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 25Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-02997421cdc55b8e4::fsap-runtime-logs + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: runtime-logs-claim + namespace: malcolm + spec: + storageClassName: efs-sc + accessModes: + - ReadWriteMany + volumeMode: Filesystem + resources: + requests: + storage: 25Gi + volumeName: runtime-logs-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: opensearch-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-opensearch + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: opensearch-claim + namespace: malcolm + spec: + storageClassName: efs-sc + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 500Gi + volumeName: opensearch-volume + + --- + apiVersion: v1 + kind: PersistentVolume + metadata: + name: opensearch-backup-volume + namespace: malcolm + labels: + namespace: malcolm + spec: + capacity: + storage: 500Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: efs-sc + csi: + driver: efs.csi.aws.com + volumeHandle: fs-FILESYSTEMID::fsap-opensearch-backup + + --- + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: opensearch-backup-claim + namespace: malcolm + spec: + storageClassName: efs-sc + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 500Gi + volumeName: opensearch-backup-volume + ``` +1. Finish [configuring](kubernetes.md#Config) and [configuring](kubernetes.md#Running) Malcolm as described in [**Deploying Malcolm with Kubernetes**](kubernetes.md) \ No newline at end of file diff --git a/docs/kubernetes.md b/docs/kubernetes.md index e743c10b5..9c7fea917 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -13,12 +13,13 @@ - [Live Traffic Analysis](#FutureLiveCap) - [Horizontal Scaling](#FutureScaleOut) - [Helm Chart](#FutureHelmChart) +* [Deploying Malcolm on Amazon Elastic Kubernetes Service (EKS)](kubernetes-eks.md#KubernetesEKS) ## System ### Ingress Controller -Malcolm's [ingress controller manifest]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/00-ingress.yml) uses the [Ingress-NGINX controller for Kubernetes](https://github.com/kubernetes/ingress-nginx). A few Malcolm features require some customization when installing and configuring the Ingress-NGINX controller: +Malcolm's [ingress controller manifest]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/00-ingress.yml) uses the [Ingress-NGINX controller for Kubernetes](https://github.com/kubernetes/ingress-nginx). A few Malcolm features require some customization when installing and configuring the Ingress-NGINX controller. As well as being listed below, see [kubernetes/vagrant/deploy_ingress_nginx.sh]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/vagrant/deploy_ingress_nginx.sh) for an example of how to configure and apply the Ingress-NGINX controller for Kubernetes. * To [forward](malcolm-hedgehog-e2e-iso-install.md#HedgehogConfigForwarding) logs from a remote instance of [Hedgehog Linux](hedgehog.md): - See ["Exposing TCP and UDP services"](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) in the Ingress-NGINX documentation. @@ -261,28 +262,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:23.05.0 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:23.05.0 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:23.05.0 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:23.05.0 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:23.05.0 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:23.05.0 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:23.05.0 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:23.05.0 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:23.05.0 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:23.05.0 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:23.05.0 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:23.05.0 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:23.05.0 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:23.05.0 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:23.05.0 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:23.05.0 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:23.05.0 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:23.05.0 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:23.05.0 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:23.05.0 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:23.05.0 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:23.05.0 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:23.05.1 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:23.05.1 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:23.05.1 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:23.05.1 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:23.05.1 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:23.05.1 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:23.05.1 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:23.05.1 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:23.05.1 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:23.05.1 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:23.05.1 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:23.05.1 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:23.05.1 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:23.05.1 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:23.05.1 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:23.05.1 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:23.05.1 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:23.05.1 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:23.05.1 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:23.05.1 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:23.05.1 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:23.05.1 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -536,28 +537,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:23.05.0 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:23.05.0 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:23.05.0 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:23.05.0 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:23.05.0 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:23.05.0 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:23.05.0 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:23.05.0 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:23.05.0 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:23.05.0 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:23.05.0 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:23.05.0 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:23.05.0 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:23.05.0 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:23.05.0 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:23.05.0 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:23.05.0 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:23.05.0 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:23.05.0 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:23.05.0 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:23.05.0 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:23.05.0 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:23.05.1 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:23.05.1 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:23.05.1 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:23.05.1 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:23.05.1 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:23.05.1 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:23.05.1 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:23.05.1 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:23.05.1 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:23.05.1 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:23.05.1 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:23.05.1 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:23.05.1 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:23.05.1 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:23.05.1 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:23.05.1 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:23.05.1 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:23.05.1 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:23.05.1 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:23.05.1 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:23.05.1 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:23.05.1 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index f2946d534..7b0f0a39a 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-23.05.0.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-23.05.1.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index 40e5d49dd..1d554f85b 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -54,25 +54,25 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 23.05.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 23.05.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 23.05.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 23.05.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 23.05.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 23.05.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 23.05.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 23.05.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 23.05.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 23.05.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 23.05.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 23.05.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 23.05.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 23.05.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 23.05.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 23.05.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 23.05.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 23.05.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 23.05.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 23.05.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 23.05.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 23.05.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 23.05.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 23.05.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 23.05.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 23.05.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 23.05.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 23.05.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 23.05.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 23.05.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 23.05.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 23.05.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 23.05.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 23.05.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 23.05.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 23.05.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 23.05.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 23.05.1 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs @@ -86,10 +86,10 @@ instance, wipe the database and restore Malcolm to a fresh state, etc. ## User interface -A few minutes after starting Malcolm (probably 5 to 10 minutes for Logstash to be completely up, depending on the system), the following services will be accessible: +A few minutes after starting Malcolm (probably 5 or so for Logstash to be completely up, depending on the system), the following services will be accessible: -* [Arkime](https://arkime.com/): [https://localhost:443](https://localhost:443) -* [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/index/): [https://localhost/dashboards/](https://localhost/dashboards/) or [https://localhost:5601](https://localhost:5601) +* [Arkime](https://arkime.com/): [https://localhost](https://localhost) +* [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/index/): [https://localhost/dashboards/](https://localhost/dashboards/) * [Capture File and Log Archive Upload (Web)](upload.md#Upload): [https://localhost/upload/](https://localhost/upload/) * [Capture File and Log Archive Upload (SFTP)](upload.md#Upload): `sftp://@127.0.0.1:8022/files` * [NetBox](asset-interaction-analysis.md#AssetInteractionAnalysis): [https://localhost/netbox/](https://localhost/netbox/) diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index 0e4e8c315..88285f207 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -256,25 +256,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 23.05.0 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 23.05.0 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 23.05.0 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 23.05.0 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 23.05.0 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 23.05.0 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 23.05.0 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 23.05.0 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 23.05.0 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 23.05.0 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 23.05.0 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 23.05.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 23.05.0 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 23.05.0 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 23.05.0 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 23.05.0 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 23.05.0 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 23.05.0 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 23.05.0 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 23.05.1 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 23.05.1 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 23.05.1 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 23.05.1 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 23.05.1 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 23.05.1 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 23.05.1 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 23.05.1 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 23.05.1 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 23.05.1 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 23.05.1 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 23.05.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 23.05.1 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 23.05.1 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 23.05.1 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 23.05.1 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 23.05.1 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 23.05.1 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 23.05.1 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. diff --git a/kubernetes/01-volumes.yml.example b/kubernetes/01-volumes.yml.example index 60555b6da..089654eeb 100644 --- a/kubernetes/01-volumes.yml.example +++ b/kubernetes/01-volumes.yml.example @@ -226,7 +226,7 @@ spec: storage: 500Gi volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Retain storageClassName: nfs mountOptions: @@ -249,7 +249,7 @@ metadata: spec: storageClassName: nfs accessModes: - - ReadWriteMany + - ReadWriteOnce volumeMode: Filesystem resources: requests: @@ -269,7 +269,7 @@ spec: storage: 500Gi volumeMode: Filesystem accessModes: - - ReadWriteMany + - ReadWriteOnce persistentVolumeReclaimPolicy: Retain storageClassName: nfs mountOptions: @@ -292,7 +292,7 @@ metadata: spec: storageClassName: nfs accessModes: - - ReadWriteMany + - ReadWriteOnce volumeMode: Filesystem resources: requests: diff --git a/kubernetes/02-opensearch.yml b/kubernetes/02-opensearch.yml index 6cf5af14e..bac0e641f 100644 --- a/kubernetes/02-opensearch.yml +++ b/kubernetes/02-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:23.05.0 + image: ghcr.io/idaholab/malcolm/opensearch:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/03-dashboards.yml b/kubernetes/03-dashboards.yml index 8db23880b..2ab9d7ad6 100644 --- a/kubernetes/03-dashboards.yml +++ b/kubernetes/03-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-upload.yml b/kubernetes/04-upload.yml index bb978dbd2..7a0233e38 100644 --- a/kubernetes/04-upload.yml +++ b/kubernetes/04-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:23.05.0 + image: ghcr.io/idaholab/malcolm/file-upload:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-pcap-monitor.yml b/kubernetes/05-pcap-monitor.yml index 04be6b978..8cb55fb9a 100644 --- a/kubernetes/05-pcap-monitor.yml +++ b/kubernetes/05-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-monitor:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-arkime.yml b/kubernetes/06-arkime.yml index 786961836..f74812290 100644 --- a/kubernetes/06-arkime.yml +++ b/kubernetes/06-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:23.05.0 + image: ghcr.io/idaholab/malcolm/arkime:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-api.yml b/kubernetes/07-api.yml index d318a8c46..33fa6d1e0 100644 --- a/kubernetes/07-api.yml +++ b/kubernetes/07-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:23.05.0 + image: ghcr.io/idaholab/malcolm/api:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-dashboards-helper.yml b/kubernetes/08-dashboards-helper.yml index bdf8cb767..4ae09aaed 100644 --- a/kubernetes/08-dashboards-helper.yml +++ b/kubernetes/08-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.0 + image: ghcr.io/idaholab/malcolm/dashboards-helper:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-zeek.yml b/kubernetes/09-zeek.yml index edfc8a64d..ab83065ca 100644 --- a/kubernetes/09-zeek.yml +++ b/kubernetes/09-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-suricata.yml b/kubernetes/10-suricata.yml index d2208ab26..d89e987a7 100644 --- a/kubernetes/10-suricata.yml +++ b/kubernetes/10-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/11-file-monitor.yml b/kubernetes/11-file-monitor.yml index 05b4227f1..4d65038a5 100644 --- a/kubernetes/11-file-monitor.yml +++ b/kubernetes/11-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:23.05.0 + image: ghcr.io/idaholab/malcolm/file-monitor:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-filebeat.yml b/kubernetes/12-filebeat.yml index aa073b19d..86e0e1c83 100644 --- a/kubernetes/12-filebeat.yml +++ b/kubernetes/12-filebeat.yml @@ -31,7 +31,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/filebeat-oss:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-logstash.yml b/kubernetes/13-logstash.yml index 56b9c256c..2a3920b40 100644 --- a/kubernetes/13-logstash.yml +++ b/kubernetes/13-logstash.yml @@ -47,7 +47,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.0 + image: ghcr.io/idaholab/malcolm/logstash-oss:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 9fceac45d..dcada13b0 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index 1096ca615..5d8ff37e9 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:23.05.0 + image: ghcr.io/idaholab/malcolm/redis:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 5d5ad21a0..70f70002f 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:23.05.0 + image: ghcr.io/idaholab/malcolm/postgresql:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index d22b3f7ac..ac53304b5 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:23.05.0 + image: ghcr.io/idaholab/malcolm/netbox:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index 0bfc8348a..918d3a8fb 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:23.05.0 + image: ghcr.io/idaholab/malcolm/htadmin:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index 5c9b21f3f..816c491dc 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.0 + image: ghcr.io/idaholab/malcolm/pcap-capture:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index f67b32625..835654692 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:23.05.0 + image: ghcr.io/idaholab/malcolm/zeek:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index d0fa77305..2b48ed2d5 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:23.05.0 + image: ghcr.io/idaholab/malcolm/suricata:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-freq.yml b/kubernetes/23-freq.yml index c515bd917..86f316139 100644 --- a/kubernetes/23-freq.yml +++ b/kubernetes/23-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:23.05.0 + image: ghcr.io/idaholab/malcolm/freq:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/99-nginx-proxy.yml b/kubernetes/99-nginx-proxy.yml index ccd1d5124..ef3f0e74a 100644 --- a/kubernetes/99-nginx-proxy.yml +++ b/kubernetes/99-nginx-proxy.yml @@ -37,7 +37,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.0 + image: ghcr.io/idaholab/malcolm/nginx-proxy:23.05.1 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/vagrant/Vagrantfile b/kubernetes/vagrant/Vagrantfile index 77e2e2b61..fefc7d6d8 100644 --- a/kubernetes/vagrant/Vagrantfile +++ b/kubernetes/vagrant/Vagrantfile @@ -13,8 +13,6 @@ end server_ip = "192.168.56.10" server_hostname = "server.k3s.internal" -load_balancer_additional_ports = "{\\\"appProtocol\\\": \\\"tcp\\\", \\\"name\\\": \\\"lumberjack\\\", \\\"port\\\": 5044, \\\"targetPort\\\": 5044, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"appProtocol\\\": \\\"tcp\\\", \\\"name\\\": \\\"tcpjson\\\", \\\"port\\\": 5045, \\\"targetPort\\\": 5045, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"appProtocol\\\": \\\"tcp\\\", \\\"name\\\": \\\"sftp\\\", \\\"port\\\": 8022, \\\"targetPort\\\": 8022, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"appProtocol\\\": \\\"tcp\\\", \\\"name\\\": \\\"opensearch\\\", \\\"port\\\": 9200, \\\"targetPort\\\": 9200, \\\"protocol\\\": \\\"TCP\\\"}" -deployment_additional_ports = "{\\\"name\\\": \\\"lumberjack\\\", \\\"containerPort\\\": 5044, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"name\\\": \\\"tcpjson\\\", \\\"containerPort\\\": 5045, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"name\\\": \\\"sftp\\\", \\\"containerPort\\\": 8022, \\\"protocol\\\": \\\"TCP\\\"}, {\\\"name\\\": \\\"opensearch\\\", \\\"containerPort\\\": 9200, \\\"protocol\\\": \\\"TCP\\\"}" agents = { "agent1" => "192.168.56.11", "agent2" => "192.168.56.12" } @@ -113,14 +111,11 @@ server_script_1 = <<-SHELL curl -sfL https://get.k3s.io | sh - echo "Waiting for k3s to start..." sleep 30 - curl -sSL -o /tmp/deploy_nginx.yaml https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.7.0/deploy/static/provider/cloud/deploy.yaml - yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--enable-ssl-passthrough", "--tcp-services-configmap=ingress-nginx/tcp-services"]' /tmp/deploy_nginx.yaml - yq -i "( select(.kind == \\"Deployment\\").spec.template.spec.containers[].args[] | select(contains(\\"/nginx-ingress-controller\\")) | parent | parent | .ports ) += [#{deployment_additional_ports}]" /tmp/deploy_nginx.yaml - yq -i "( select(.kind == \\"Service\\" and .spec.type == \\"LoadBalancer\\").spec.ports ) += [#{load_balancer_additional_ports}]" /tmp/deploy_nginx.yaml - kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml apply -f /tmp/deploy_nginx.yaml + bash /tmp/deploy_ingress_nginx.sh -s -t -k /etc/rancher/k3s/k3s.yaml until [ -f /var/lib/rancher/k3s/server/token ] && [ -f /etc/rancher/k3s/k3s.yaml ]; do sleep 5; done cp -v /var/lib/rancher/k3s/server/token /vagrant_shared cp -v /etc/rancher/k3s/k3s.yaml /vagrant_shared + rm -f /tmp/deploy_ingress_nginx.sh SHELL agent_script_1 = <<-SHELL @@ -168,6 +163,7 @@ Vagrant.configure("2") do |config| server.vm.provision "shell", inline: server_script_0 server.vm.provision "shell", inline: common_script_0 server.vm.provision :reload + server.vm.provision "file", source: "./deploy_ingress_nginx.sh", destination: "/tmp/deploy_ingress_nginx.sh" server.vm.provision "shell", inline: server_script_1 end diff --git a/kubernetes/vagrant/deploy_ingress_nginx.sh b/kubernetes/vagrant/deploy_ingress_nginx.sh new file mode 100755 index 000000000..b9d17be93 --- /dev/null +++ b/kubernetes/vagrant/deploy_ingress_nginx.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +if [ -z "$BASH_VERSION" ]; then + echo "Wrong interpreter, please run \"$0\" with bash" >&2 + exit 1 +fi + +############################################################################### +# script options +set -o pipefail +set -e +shopt -s nocasematch +ENCODING="utf-8" + +############################################################################### +# script variables +LOAD_BALANCER_ADDITIONAL_PORTS="{\"appProtocol\": \"tcp\", \"name\": \"lumberjack\", \"port\": 5044, \"targetPort\": 5044, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"tcpjson\", \"port\": 5045, \"targetPort\": 5045, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"sftp\", \"port\": 8022, \"targetPort\": 8022, \"protocol\": \"TCP\"}, {\"appProtocol\": \"tcp\", \"name\": \"opensearch\", \"port\": 9200, \"targetPort\": 9200, \"protocol\": \"TCP\"}" +DEPLOYMENT_ADDITIONAL_PORTS="{\"name\": \"lumberjack\", \"containerPort\": 5044, \"protocol\": \"TCP\"}, {\"name\": \"tcpjson\", \"containerPort\": 5045, \"protocol\": \"TCP\"}, {\"name\": \"sftp\", \"containerPort\": 8022, \"protocol\": \"TCP\"}, {\"name\": \"opensearch\", \"containerPort\": 9200, \"protocol\": \"TCP\"}" +AWS_EXPOSE_ANNOTATIONS=( + # see https://repost.aws/knowledge-center/eks-access-kubernetes-services (Option 1), step 2. + "{\"service.beta.kubernetes.io/aws-load-balancer-backend-protocol\":\"tcp\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled\":\"true\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-type\":\"external\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type\":\"instance\"}" + "{\"service.beta.kubernetes.io/aws-load-balancer-scheme\":\"internet-facing\"}" +) +INGRESS_NGINX_CONTROLLER_VERSION=1.7.0 +KUBECONFIG= +WORKDIR= +DRY_RUN=none +INGRESS_NGINX_PROVIDER=cloud +EXPOSE_VIA_AWS_LB= +SSL_PASSTHROUGH= +OTHER_TCP_SERVICES= + +############################################################################### +# show script usage +function help() { + echo -e "\n$(basename $0)\n" + echo -e "-h display help\n" + echo -e "-v enable bash verbosity\n" + echo -e "-k kubeconfig kubeconfig file\n" + echo -e "-d dryrunval --dry-run=dryrunval for kubectl apply (none|server|client)\n" + echo -e "-i version ingress-nginx controller version" + echo -e " https://github.com/kubernetes/ingress-nginx/releases\n" + echo -e "-a use AWS provider for ingress-nginx" + echo -e " OR" + echo -e "-p provider specify provider for ingress-nginx" + echo -e " https://github.com/kubernetes/ingress-nginx/tree/main/deploy/static/provider\n" + echo -e "-e expose ingress-nginx via AWS load balancer (only applies to -a/-p aws)" + echo -e " https://repost.aws/knowledge-center/eks-access-kubernetes-services\n" + echo -e "-s start ingress-nginx with --enable-ssl-passthrough" + echo -e " https://kubernetes.github.io/ingress-nginx/user-guide/tls/#ssl-passthrough\n" + echo -e "-t start ingress-nginx with --tcp-services-configmap=ingress-nginx/tcp-services" + echo -e " https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services\n" + exit 1 +} + +############################################################################### +# parse command-line parameters +while getopts 'vhaestp:d:k:i:' OPTION; do + case "$OPTION" in + + v) + VERBOSE_FLAG="-v" + # set -x + ;; + + d) + DRY_RUN="${OPTARG}" + ;; + + p) + INGRESS_NGINX_PROVIDER="${OPTARG}" + ;; + + a) + INGRESS_NGINX_PROVIDER="aws" + ;; + + e) + EXPOSE_VIA_AWS_LB="true" + ;; + + s) + SSL_PASSTHROUGH="true" + ;; + + t) + OTHER_TCP_SERVICES="true" + ;; + + k) + KUBECONFIG="${OPTARG}" + ;; + + i) + INGRESS_NGINX_CONTROLLER_VERSION="${OPTARG}" + ;; + + ?) + help >&2 + exit 1; + ;; + + esac +done +shift "$(($OPTIND -1))" + +############################################################################### +function cleanup { + set +e + if [[ -n "${WORKDIR}" ]] && [[ -d "${WORKDIR}" ]]; then + popd >/dev/null >/dev/null 2>&1 + rm ${VERBOSE_FLAG} -r -f "${WORKDIR}" >/dev/null 2>&1 + fi +} + +if ! command -v curl >/dev/null 2>&1 || ! command -v yq >/dev/null 2>&1 || ! command -v kubectl >/dev/null 2>&1; then + echo "$(basename $0) requires curl, kubectl and yq" >&2 + exit 1 + +elif [[ -z "${KUBECONFIG}" ]] || [[ ! -f "${KUBECONFIG}" ]]; then + echo "$(basename $0) requires kubeconfig specified with -k" >&2 + exit 1 +fi + +############################################################################### + +trap "cleanup" EXIT + +WORKDIR="$(mktemp -d -t malcolm-XXXXXX)" +pushd "${WORKDIR}" >/dev/null 2>&1 + +INGRESS_NGINX_DEPLOY_FILE_ORIG=ingress-nginx-orig.yaml +INGRESS_NGINX_DEPLOY_FILE_NEW=ingress-nginx-new.yaml + +curl -fsSL -o "${INGRESS_NGINX_DEPLOY_FILE_ORIG}" "https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v${INGRESS_NGINX_CONTROLLER_VERSION}/deploy/static/provider/${INGRESS_NGINX_PROVIDER}/deploy.yaml" +yq --split-exp '"deploy_" + $index' --no-doc "${INGRESS_NGINX_DEPLOY_FILE_ORIG}" + +readarray -d '' DEPLOY_FILES_SPLIT < <(printf '%s\0' deploy_*.yml | sort -zV) +for DEPLOY_FILE in "${DEPLOY_FILES_SPLIT[@]}"; do + + if (( $(yq 'select(.kind == "Deployment")' "${DEPLOY_FILE}" | wc -l) > 0 )); then + + if [[ "${SSL_PASSTHROUGH}" == "true" ]]; then + yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--enable-ssl-passthrough"]' "${DEPLOY_FILE}" + fi + + if [[ "${OTHER_TCP_SERVICES}" == "true" ]]; then + yq -i '( select(.kind == "Deployment").spec.template.spec.containers[].args[] | select(contains("/nginx-ingress-controller")) | parent ) += ["--tcp-services-configmap=ingress-nginx/tcp-services"]' "${DEPLOY_FILE}" + yq -i "( select(.kind == \"Deployment\").spec.template.spec.containers[].args[] | select(contains(\"/nginx-ingress-controller\")) | parent | parent | .ports ) += [${DEPLOYMENT_ADDITIONAL_PORTS}]" "${DEPLOY_FILE}" + fi + fi + + if (( $(yq 'select(.kind == "Service" and .spec.type == "LoadBalancer")' "${DEPLOY_FILE}" | wc -l) > 0 )); then + + if [[ "${OTHER_TCP_SERVICES}" == "true" ]]; then + yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").spec.ports ) += [${LOAD_BALANCER_ADDITIONAL_PORTS}]" "${DEPLOY_FILE}" + fi + + if [[ "${EXPOSE_VIA_AWS_LB}" == "true" ]]; then + # see https://repost.aws/knowledge-center/eks-access-kubernetes-services (Option 1), step 2. + for OLDKEY in $(yq "select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").metadata.annotations | keys | .[] | select(. == \"service.beta.kubernetes.io*\")" "${DEPLOY_FILE}"); do + yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\") ) | del(.metadata.annotations.\"$OLDKEY\")" "${DEPLOY_FILE}" + done + for NEWKEY in ${AWS_EXPOSE_ANNOTATIONS[@]}; do + yq -i "( select(.kind == \"Service\" and .spec.type == \"LoadBalancer\").metadata.annotations ) += ${NEWKEY}" "${DEPLOY_FILE}" + done + fi + fi + + [[ -f "${INGRESS_NGINX_DEPLOY_FILE_NEW}" ]] && echo "---" >> "${INGRESS_NGINX_DEPLOY_FILE_NEW}" + cat "${DEPLOY_FILE}" >> "${INGRESS_NGINX_DEPLOY_FILE_NEW}" + +done + +[[ -n "${VERBOSE_FLAG}" ]] && cat "${INGRESS_NGINX_DEPLOY_FILE_NEW}" + +kubectl --kubeconfig "${KUBECONFIG}" apply --dry-run="${DRY_RUN}" -f "${INGRESS_NGINX_DEPLOY_FILE_NEW}" + +exit 0 \ No newline at end of file diff --git a/malcolm-iso/config/includes.chroot/etc/bash.bash_functions b/malcolm-iso/config/includes.chroot/etc/bash.bash_functions index f28514ab8..bfc6b2487 100644 --- a/malcolm-iso/config/includes.chroot/etc/bash.bash_functions +++ b/malcolm-iso/config/includes.chroot/etc/bash.bash_functions @@ -223,6 +223,30 @@ function h() { if [ -z "$1" ]; then history; else history | grep -i "$@"; fi; } ######################################################################## function fname() { find . -iname "*$@*"; } +function findupes() { + find . -not -empty -type f -printf "%s\n" 2>/dev/null | \ + sort -rn | \ + uniq -d | \ + xargs -I{} -n1 find -type f -size {}c -print0 | \ + xargs -0 md5sum | \ + sort | \ + uniq -w32 --all-repeated=separate +} + +function sfind() { + if [ "$1" ]; then + FIND_FOLDER="$1" + else + FIND_FOLDER="$(pwd)" + fi + if [ "$2" ]; then + FIND_PATTERN="$2" + else + FIND_PATTERN="*" + fi + find "$FIND_FOLDER" -type f -iname "$FIND_PATTERN" -print0 | xargs -r -0 ls -la | awk '{system("numfmt -z --to=iec-i --suffix=B --padding=7 "$5) ; out=""; for(i=9;i<=NF;i++){out=out" "$i}; print " KB\t"out}' | sort -h +} + ######################################################################## # examine running processes ######################################################################## @@ -291,7 +315,36 @@ function arps() function portping() { - python <<<"import socket; socket.setdefaulttimeout(1); socket.socket().connect(('$1', $2))" 2> /dev/null && echo OPEN || echo CLOSED; + CONN_TIMEOUT=5 + if [[ -n "$BASH_VERSION" ]] && [[ $LINUX ]]; then + # use /dev/tcp + timeout $CONN_TIMEOUT bash -c "cat /dev/null > /dev/tcp/$1/$2" && echo OPEN || echo CLOSED + elif command -v python3 >/dev/null 2>&1; then + # use python socket library + python3 <<<"import socket; socket.setdefaulttimeout($CONN_TIMEOUT); socket.socket().connect(('$1', $2))" 2> /dev/null && echo OPEN || echo CLOSED + elif command -v socat >/dev/null 2>&1; then + # use socat + socat /dev/null TCP4:"$1":"$2",connect-timeout="$CONN_TIMEOUT" >/dev/null 2>&1 && echo OPEN || echo CLOSED + elif command -v nc >/dev/null 2>&1; then + # use some flavor of netcat + if ( nc -h 2>&1 | grep -q 'to somewhere' ); then + # traditional + ( timeout $((CONN_TIMEOUT+1)) bash -c "cat /dev/null | nc -v -w "$CONN_TIMEOUT" "$1" "$2" 2>&1" || true ) | grep -q 'open$' && echo OPEN || echo CLOSED + elif ( nc 2>&1 | grep -q '46CDdFhklNnrStUuvZz' ); then + # openbsd + timeout $((CONN_TIMEOUT+1)) bash -c "cat /dev/null | nc -w "$CONN_TIMEOUT" "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + elif ( nc --help 2>&1 | grep -q 'Ncat' ); then + # ncat + timeout $CONN_TIMEOUT bash -c "cat /dev/null | nc -v --send-only "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + else + echo UNKNOWN + fi + elif command -v telnet >/dev/null 2>&1; then + # use telnet + timeout $CONN_TIMEOUT bash -c "echo -e '\x1dclose\x0d' | telnet "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + else + echo UNKNOWN + fi } ######################################################################## @@ -443,7 +496,7 @@ alias dis="docker images | tail -n +2 | cols 1 2 | sed \"s/ /:/\"" alias dip="docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'" # a slimmed-down stats -alias dstats="docker stats --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}\t{{.BlockIO}}'" +alias dstats="docker stats --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}'" # Execute in existing interactive container, e.g., $dex base /bin/bash alias dex="docker exec -i -t" @@ -498,16 +551,16 @@ function malcolmmonitor () { split-window -v \; \ split-window -v \; \ select-pane -t 1 \; \ - send-keys '~/Malcolm/scripts/logs' C-m \; \ + send-keys 'pushd ~/Malcolm >/dev/null 2>&1; ~/Malcolm/scripts/logs; popd >/dev/null 2>&1' C-m \; \ select-pane -t 2 \; \ - send-keys 'dstats' C-m \; \ + send-keys "docker stats --format 'table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}'" C-m \; \ select-pane -t 3 \; \ - send-keys 'while true; do clear; df -h ~/Malcolm/; sleep 60; done' C-m \; \ + send-keys 'while true; do clear; df -h ~/Malcolm/ | tail -n +2; sleep 60; done' C-m \; \ select-pane -t 4 \; \ send-keys 'top' C-m \; \ split-window -v \; \ select-pane -t 5 \; \ - send-keys 'while true; do clear; free -m | head -n 2; sleep 60; done' C-m \; \ + send-keys 'while true; do clear; free -m | grep ^Mem: | cut -d" " -f2- | sed "s/[[:space:]]\+/,/g" | sed "s/^,//" ; sleep 60; done' C-m \; \ select-pane -t 6 \; \ send-keys "while true; do clear; pushd ~/Malcolm >/dev/null 2>&1; docker-compose exec -u $(id -u) api curl -sSL 'http://localhost:5000/mapi/agg/event.dataset?from=1970' | python3 -m json.tool | grep -P '\b(doc_count|key)\b' | tr -d '\", ' | cut -d: -f2 | paste - - -d'\t\t' | head -n $(( (MAX_HEIGHT / 2) - 1 )) ; popd >/dev/null 2>&1; sleep 60; done" C-m \; \ select-pane -t 7 \; \ @@ -519,14 +572,15 @@ function malcolmmonitor () { send-keys "while true; do clear; find ~/Malcolm/zeek-logs/extract_files -type f | sed 's@.*/@@' | sed 's/.*\.//' | sort | uniq -c | sort -nr | head -n $(( (MAX_HEIGHT / 3) - 1 )) ; sleep 60; done" C-m \; \ select-pane -t 9 \; \ resize-pane -R $(( ($MAX_WIDTH / 2) - 30 )) \; \ + select-pane -t 1 \; \ + resize-pane -D 999 \; \ + resize-pane -U 24 \; \ select-pane -t 3 \; \ - resize-pane -D $(( ($MAX_HEIGHT / 4) - 4 )) \; \ + resize-pane -D 999 \; \ + resize-pane -U 1 \; \ select-pane -t 5 \; \ - resize-pane -D $(( ($MAX_HEIGHT / 4) - 4 )) \; \ - select-pane -t 7 \; \ - resize-pane -U $(( ($MAX_HEIGHT / 8) - 4 )) \; \ - select-pane -t 8 \; \ - resize-pane -U $(( ($MAX_HEIGHT / 8) - 1 )) \; \ - select-pane -t 4 \; + resize-pane -D 999 \; \ + resize-pane -U 1 \; fi } + diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.bashrc b/malcolm-iso/config/includes.chroot/etc/skel/.bashrc index a4b80d247..a18e760e7 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.bashrc +++ b/malcolm-iso/config/includes.chroot/etc/skel/.bashrc @@ -40,6 +40,10 @@ fi ############################################################################### # PATH ############################################################################### +if [ -d /opt/fluent-bit/bin ]; then + PATH=/opt/fluent-bit/bin:$PATH +fi + if [ -d ~/bin ]; then PATH=~/bin:$PATH fi diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/procps/toprc b/malcolm-iso/config/includes.chroot/etc/skel/.config/procps/toprc new file mode 100644 index 000000000..481147027 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/procps/toprc @@ -0,0 +1,16 @@ +top's Config File (Linux processes with windows) +Id:j, Mode_altscr=0, Mode_irixps=1, Delay_time=1.0, Curwin=0 +Def fieldscur=¥(34»½@Ä·º¹Å&')*+,-./012568<>?ABCFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz + winflags=193844, sortindx=18, maxtasks=0, graph_cpus=0, graph_mems=0, double_up=0, combine_cpus=0 + summclr=1, msgsclr=1, headclr=3, taskclr=1 +Job fieldscur=¥¦¹·º(³´Ä»½@<§Å)*+,-./012568>?ABCFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz + winflags=193844, sortindx=0, maxtasks=0, graph_cpus=0, graph_mems=0, double_up=0, combine_cpus=0 + summclr=6, msgsclr=6, headclr=7, taskclr=6 +Mem fieldscur=¥º»<½¾¿ÀÁMBNÃD34·Å&'()*+,-./0125689FGHIJKLOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz + winflags=193844, sortindx=21, maxtasks=0, graph_cpus=0, graph_mems=0, double_up=0, combine_cpus=0 + summclr=5, msgsclr=5, headclr=4, taskclr=5 +Usr fieldscur=¥¦§¨ª°¹·ºÄÅ)+,-./1234568;<=>?@ABCFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz + winflags=193844, sortindx=3, maxtasks=0, graph_cpus=0, graph_mems=0, double_up=0, combine_cpus=0 + summclr=3, msgsclr=3, headclr=2, taskclr=3 +Fixed_widest=0, Summ_mscale=1, Task_mscale=0, Zero_suppress=0 + diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service new file mode 100644 index 000000000..bf7bc40c6 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/cpu-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i cpu -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=cpu -p WildCard='*' -m '*' -F record_modifier -p 'Record=module cpu' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service new file mode 100644 index 000000000..3ece47d60 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/df-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Parser=json -p Command=/usr/local/bin/df-json.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=disk -p WildCard='*' -m '*' -F record_modifier -p 'Record=module disk' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service new file mode 100644 index 000000000..8fb2fee1b --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/disk-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i disk -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=disk -p WildCard='*' -m '*' -F record_modifier -p 'Record=module disk' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service new file mode 100644 index 000000000..f73368f78 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/mem-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i mem -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=mem -p WildCard='*' -m '*' -F record_modifier -p 'Record=module mem' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service new file mode 100644 index 000000000..f9d8e9135 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/memp-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Command=/usr/local/bin/memory_usage_percentage.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F modify -p "Hard_rename=exec Mem.used_p" -m '*' -F nest -p Operation=nest -p Nested_under=mem -p WildCard='*' -m '*' -F record_modifier -p 'Record=module mem' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service new file mode 100644 index 000000000..e0a1cf718 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/network-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i exec -p Parser=json -p Command=/usr/local/bin/netdev-json.sh -p Interval_Sec=30 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=network -p WildCard='*' -m '*' -F record_modifier -p 'Record=module network' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service new file mode 100644 index 000000000..6ea73ba54 --- /dev/null +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/systemd/user/thermal-localhost-malcolm.service @@ -0,0 +1,12 @@ +[Unit] +AssertPathExists=%h/Malcolm/filebeat/certs/client.key +After=network.target + +[Service] +ExecStart=/opt/fluent-bit/bin/fluent-bit -R /etc/fluent-bit/parsers.conf -i thermal -p Interval_Sec=10 -o tcp://localhost:5045 -p tls=on -p tls.verify=off -p tls.ca_file=%h/Malcolm/filebeat/certs/ca.crt -p tls.crt_file=%h/Malcolm/filebeat/certs/client.crt -p tls.key_file=%h/Malcolm/filebeat/certs/client.key -p format=json_lines -F nest -p Operation=nest -p Nested_under=thermal -p WildCard='*' -m '*' -F record_modifier -p 'Record=module thermal' -m '*' -f 1 +Restart=on-failure +PrivateTmp=false +NoNewPrivileges=false + +[Install] +WantedBy=default.target diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-25/16343171677.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-25/16343171677.desktop index aead73d5a..a20408949 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-25/16343171677.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-25/16343171677.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Start Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --start" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --start" Comment=Start Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-26/16343171699.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-26/16343171699.desktop index 854b12df3..9074a72fb 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-26/16343171699.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-26/16343171699.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Restart Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --restart" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --restart" Comment=Restart Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-27/16343171722.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-27/16343171722.desktop index 87de10b44..e3a97a508 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-27/16343171722.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-27/16343171722.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Stop Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --stop" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --stop" Comment=Stop Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-28/16343171811.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-28/16343171811.desktop index b67b49d66..bffcb003d 100644 --- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-28/16343171811.desktop +++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-28/16343171811.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Malcolm Debug Logs -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --logs" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --logs" Comment=Monitor the debug output of Malcolm containers Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-logs.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-logs.desktop index cb3660de9..5ea913299 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-logs.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-logs.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Malcolm Debug Logs -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --logs" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --logs" Comment=Monitor the debug output of Malcolm containers Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-restart.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-restart.desktop index 6c0f0a06d..1194f84c1 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-restart.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-restart.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Restart Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --restart" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --restart" Comment=Restart Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-start.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-start.desktop index 007e8e8c5..39301d22b 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-start.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-start.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Start Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --start" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --start" Comment=Start Malcolm Terminal=false Type=Application diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-stop.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-stop.desktop index ac18f0e3c..53bb34ef1 100644 --- a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-stop.desktop +++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-stop.desktop @@ -1,6 +1,6 @@ [Desktop Entry] Name=Stop Malcolm -Exec=tilix -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/control.py --stop" +Exec=tilix -e /bin/bash -l -c "cd ~/Malcolm && /usr/bin/python3 ~/Malcolm/scripts/control.py --stop" Comment=Stop Malcolm Terminal=false Type=Application diff --git a/scripts/control.py b/scripts/control.py index d82df1738..2eea83cb1 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -3,6 +3,10 @@ # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +import sys + +sys.dont_write_bytecode = True + import argparse import errno import fileinput @@ -18,7 +22,6 @@ import signal import stat import string -import sys import tarfile import time @@ -395,17 +398,19 @@ def status(): else: eprint("Failed to display Malcolm status\n") eprint("\n".join(out)) - exit(err) elif orchMode is OrchestrationFramework.KUBERNETES: try: PrintNodeStatus() print() + except Exception as e: + if args.debug: + eprint(f'Error getting node status: {e}') + try: PrintPodStatus(namespace=args.namespace) print() except Exception as e: eprint(f'Error getting {args.namespace} status: {e}') - exit(-1) else: raise Exception(f'{sys._getframe().f_code.co_name} does not yet support {orchMode}') diff --git a/scripts/install.py b/scripts/install.py index 48e1596b8..aa2fbea38 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -3,6 +3,10 @@ # Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. +import sys + +sys.dont_write_bytecode = True + import argparse import datetime import errno @@ -17,7 +21,6 @@ import math import re import shutil -import sys import tarfile import tempfile import time @@ -405,8 +408,14 @@ def tweak_malcolm_runtime( raise Exception("Could not determine configuration directory containing Malcolm's .env files") # figure out what UID/GID to run non-rood processes under docker as - puid = '1000' - pgid = '1000' + defaultUid = '1000' + defaultGid = '1000' + if ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) and (self.scriptUser == "root"): + defaultUid = str(os.stat(malcolm_install_path).st_uid) + defaultGid = str(os.stat(malcolm_install_path).st_gid) + + puid = defaultUid + pgid = defaultGid try: if self.platform == PLATFORM_LINUX: puid = str(os.getuid()) @@ -414,8 +423,8 @@ def tweak_malcolm_runtime( if (puid == '0') or (pgid == '0'): raise Exception('it is preferrable not to run Malcolm as root, prompting for UID/GID instead') except Exception: - puid = '1000' - pgid = '1000' + puid = defaultUid + pgid = defaultGid while ( (not puid.isdigit()) @@ -438,21 +447,21 @@ def tweak_malcolm_runtime( ) if self.totalMemoryGigs >= 63.0: - osMemory = '30g' - lsMemory = '6g' + osMemory = '24g' + lsMemory = '3g' elif self.totalMemoryGigs >= 31.0: osMemory = '16g' - lsMemory = '3g' + lsMemory = '2500m' elif self.totalMemoryGigs >= 15.0: osMemory = '10g' lsMemory = '2500m' elif self.totalMemoryGigs >= 11.0: osMemory = '6g' - lsMemory = '2500m' + lsMemory = '2g' elif self.totalMemoryGigs >= 7.0: eprint(f"Detected only {self.totalMemoryGigs} GiB of memory; performance will be suboptimal") osMemory = '4g' - lsMemory = '2500m' + lsMemory = '2g' elif self.totalMemoryGigs > 0.0: eprint(f"Detected only {self.totalMemoryGigs} GiB of memory; performance will be suboptimal") osMemory = '3500m' @@ -472,9 +481,9 @@ def tweak_malcolm_runtime( # we don't want it too high, as in Malcolm Logstash also competes with OpenSearch, etc. for resources if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: if self.totalCores > 16: - lsWorkers = 10 - elif self.totalCores >= 12: lsWorkers = 6 + elif self.totalCores >= 12: + lsWorkers = 4 else: lsWorkers = 3 else: @@ -975,28 +984,6 @@ def tweak_malcolm_runtime( if not os.path.isfile(envFile): shutil.copyfile(envExampleFile, envFile) - # change ownership of .envs file to match puid/pgid - if ( - ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) - and (self.scriptUser == "root") - and (getpwuid(os.stat(args.configDir).st_uid).pw_name == self.scriptUser) - ): - if args.debug: - eprint(f"Setting permissions of {args.configDir} to {puid}:{pgid}") - os.chown(args.configDir, int(puid), int(pgid)) - envFiles = [] - for exts in ('*.env', '*.env.example'): - envFiles.extend(glob.glob(os.path.join(args.configDir, exts))) - for envFile in envFiles: - if ( - ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) - and (self.scriptUser == "root") - and (getpwuid(os.stat(envFile).st_uid).pw_name == self.scriptUser) - ): - if args.debug: - eprint(f"Setting permissions of {envFile} to {puid}:{pgid}") - os.chown(envFile, int(puid), int(pgid)) - # define environment variables to be set in .env files EnvValue = namedtuple("EnvValue", ["envFile", "key", "value"], rename=False) @@ -1375,15 +1362,50 @@ def tweak_malcolm_runtime( pass try: - dotenv_imported.set_key( - val.envFile, - val.key, - val.value, - quote_mode='never', - encoding='utf-8', - ) + oldDotEnvVersion = False + try: + dotenv_imported.set_key( + val.envFile, + val.key, + str(val.value), + quote_mode='never', + encoding='utf-8', + ) + except TypeError: + oldDotEnvVersion = True + + if oldDotEnvVersion: + dotenv_imported.set_key( + val.envFile, + val.key, + str(val.value), + quote_mode='never', + ) + except Exception as e: - eprint(f"Setting value for {val.key} in {val.envFile} module failed: {e}") + eprint(f"Setting value for {val.key} in {val.envFile} module failed ({type(e).__name__}): {e}") + + # change ownership of .envs file to match puid/pgid + if ( + ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) + and (self.scriptUser == "root") + and (getpwuid(os.stat(args.configDir).st_uid).pw_name == self.scriptUser) + ): + if args.debug: + eprint(f"Setting permissions of {args.configDir} to {puid}:{pgid}") + os.chown(args.configDir, int(puid), int(pgid)) + envFiles = [] + for exts in ('*.env', '*.env.example'): + envFiles.extend(glob.glob(os.path.join(args.configDir, exts))) + for envFile in envFiles: + if ( + ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) + and (self.scriptUser == "root") + and (getpwuid(os.stat(envFile).st_uid).pw_name == self.scriptUser) + ): + if args.debug: + eprint(f"Setting permissions of {envFile} to {puid}:{pgid}") + os.chown(envFile, int(puid), int(pgid)) if self.orchMode is OrchestrationFramework.DOCKER_COMPOSE: # modify docker-compose specific values (port mappings, volume bind mounts, etc.) in-place in docker-compose files @@ -1744,6 +1766,8 @@ def tweak_malcolm_runtime( try: touch(MalcolmCfgRunOnceFile) + if ((self.platform == PLATFORM_LINUX) or (self.platform == PLATFORM_MAC)) and (self.scriptUser == "root"): + os.chown(MalcolmCfgRunOnceFile, int(puid), int(pgid)) except Exception: pass diff --git a/scripts/malcolm_common.py b/scripts/malcolm_common.py index 8dfb84a3b..50ba6223d 100644 --- a/scripts/malcolm_common.py +++ b/scripts/malcolm_common.py @@ -678,6 +678,7 @@ def DownloadToFile(url, local_filename, debug=False): | eshealth | esindices/list | executing\s+attempt_(transition|set_replica_count)\s+for + | failed\s+to\s+get\s+tcp\s+stats\s+from\s+/proc | GET\s+/(netbox/api|_cat/health|api/status|sessions2-|arkime_\w+).+HTTP/[\d\.].+\b200\b | loaded\s+config\s+'/etc/netbox/config/ | "netbox"\s+application\s+started diff --git a/sensor-iso/arkime/Dockerfile b/sensor-iso/arkime/Dockerfile index eaec223e1..1884a9d56 100644 --- a/sensor-iso/arkime/Dockerfile +++ b/sensor-iso/arkime/Dockerfile @@ -6,7 +6,7 @@ LABEL maintainer="malcolm@inl.gov" ENV DEBIAN_FRONTEND noninteractive -ENV ARKIME_VERSION "4.3.0" +ENV ARKIME_VERSION "4.3.1" ENV ARKIME_DIR "/opt/arkime" RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.list && \ diff --git a/sensor-iso/beats/Dockerfile b/sensor-iso/beats/Dockerfile deleted file mode 100644 index 5126c1bb3..000000000 --- a/sensor-iso/beats/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -FROM debian:buster-slim - -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. - -LABEL maintainer="malcolm@inl.gov" - -ENV DEBIAN_FRONTEND noninteractive -ENV GOPATH=/go -ENV GOBIN=/go/bin -ENV GOARCH=amd64 -ENV GOVERS="2:1.15~1~bpo10+1" -ENV PATH="$GOBIN:${PATH}" -ENV PYTHON_EXE=python3 - -RUN set -x && \ - sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \ - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \ - apt-get -q update && \ - apt-get install -y curl git vim-tiny && \ - apt-get install -t buster-backports -y \ - "golang-doc=$GOVERS" \ - "golang-go=$GOVERS" \ - "golang-src=$GOVERS" \ - "golang=$GOVERS" \ - build-essential \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-virtualenv \ - python3-wheel \ - virtualenv && \ - rm -rf /var/lib/apt/lists/* && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3 2 && \ - update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 2 && \ - python3 -m pip install -U pyyaml cookiecutter && \ - mkdir -p "$GOPATH/bin" && \ - bash -c "curl -sSL https://raw.githubusercontent.com/Masterminds/glide.sh/master/get | sed 's@https://glide.sh/@https://raw.githubusercontent.com/Masterminds/glide.sh/master/@g'| bash" && \ - go get -u -d github.com/magefile/mage && \ - cd $GOPATH/src/github.com/magefile/mage && \ - go run bootstrap.go - -ENV BEATS=filebeat -ENV BEATS_VERSION=8.6.2 - -ADD ./build.sh /build.sh -RUN [ "chmod", "+x", "/build.sh" ] -RUN [ "mkdir", "-p", "/go" ] -RUN [ "mkdir", "/build" ] - -CMD "/build.sh" diff --git a/sensor-iso/beats/beat-build.sh b/sensor-iso/beats/beat-build.sh deleted file mode 100755 index 63ada694c..000000000 --- a/sensor-iso/beats/beat-build.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. - -VERSION="8.6.0" -THIRD_PARTY_BRANCH="master" -while getopts b:v:t: opts; do - case ${opts} in - b) BEAT=${OPTARG} ;; - v) VERSION=${OPTARG} ;; - t) THIRD_PARTY_BRANCH=${OPTARG} ;; - esac -done - -if [[ -z $BEAT || -z $VERSION || -z $THIRD_PARTY_BRANCH ]] ; then - echo "usage:" >&2 - echo " beat-build.sh -b [-v ] [-v ]" >&2 - echo "" >&2 - echo "example:" >&2 - echo " beat-build.sh -b filebeat -v $VERSION" >&2 - exit 1 -fi - -BEAT_DIR="$(pwd)/$(echo "$BEAT" | sed "s@^https*://@@" | sed 's@/@_@g')" -mkdir -p "$BEAT_DIR" -docker run --rm -v "$BEAT_DIR":/build -e "BEATS_VERSION=$VERSION" -e "THIRD_PARTY_BRANCH=$THIRD_PARTY_BRANCH" -e "BEATS=$BEAT" beats-build:latest diff --git a/sensor-iso/beats/build-docker-image.sh b/sensor-iso/beats/build-docker-image.sh deleted file mode 100755 index ef9cb305d..000000000 --- a/sensor-iso/beats/build-docker-image.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. - -# force-navigate to script directory -SCRIPT_PATH="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd "$SCRIPT_PATH" >/dev/null 2>&1 - -docker build -t beats-build:latest . - -popd >/dev/null 2>&1 diff --git a/sensor-iso/beats/build.sh b/sensor-iso/beats/build.sh deleted file mode 100755 index 87da7e31d..000000000 --- a/sensor-iso/beats/build.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved. - -echo Target version: $BEATS_VERSION - -BRANCH=$(echo $BEATS_VERSION | awk -F \. {'print $1 "." $2'}) -echo Target branch: $BRANCH - -if [ ! -d "$GOPATH/src/github.com/elastic/beats" ]; then go get -v github.com/elastic/beats; fi - -cd $GOPATH/src/github.com/elastic/beats -git checkout $BRANCH - -IFS="," -BEATS_ARRAY=($BEATS) - -for BEAT in "${BEATS_ARRAY[@]}" -do - - if [[ -d "$GOPATH/src/github.com/elastic/beats/$BEAT" ]] ; then - # an official beat - cd "$GOPATH/src/github.com/elastic/beats/$BEAT" - make - cp "$BEAT" /build - - # package - DOWNLOAD="$BEAT-$BEATS_VERSION-linux-x86.tar.gz" - if [ ! -e $DOWNLOAD ]; then curl -s -O -J "https://artifacts.elastic.co/downloads/beats/$BEAT/$DOWNLOAD"; fi - tar xf "$DOWNLOAD" - - cp "$BEAT" "$BEAT-$BEATS_VERSION-linux-x86" - tar zcf "$BEAT-$BEATS_VERSION-linux-amd64.tar.gz" "$BEAT-$BEATS_VERSION-linux-x86" - cp "$BEAT-$BEATS_VERSION-linux-amd64.tar.gz" /build - - elif [[ "$BEAT" =~ ^https*://(gogs\..*|github\.com) ]] ; then - BRANCH=${THIRD_PARTY_BRANCH:-"master"} - - # clone from git manually rather than do a "go get" - mkdir -p "$GOPATH/src/$(dirname "$(echo "$BEAT" | sed "s@^https*://@@")")" - cd "$GOPATH/src/$(dirname "$(echo "$BEAT" | sed "s@^https*://@@")")" - git clone --depth=1 --single-branch --branch "$BRANCH" "$BEAT" - BEAT_EXE_NAME="$(basename "$BEAT" | sed "s/\.git$//")" - cd "$BEAT_EXE_NAME" - go get - go install - if [[ -f "$GOBIN/$BEAT_EXE_NAME" ]] ; then - cp "$GOBIN/$BEAT_EXE_NAME" /build - strip "/build/$BEAT_EXE_NAME" - fi - - else - # a community beat? - if [[ "$BEAT" =~ gogs\..* ]]; then - INSECURE_FLAG="--insecure" - else - INSECURE_FLAG="" - fi - go get $INSECURE_FLAG "$BEAT" - BEAT_EXE_NAME="$(basename "$BEAT")" - if [[ -f "$GOBIN/$BEAT_EXE_NAME" ]] ; then - cp "$GOBIN/$BEAT_EXE_NAME" /build - strip "/build/$BEAT_EXE_NAME" - fi - fi - - ls -lh /build - -done diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot index d9fdc9deb..6af459b13 100755 --- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -20,7 +20,7 @@ export PATH="${ZEEK_DIR}"/bin:$PATH SURICATA_RULES_DIR="/etc/suricata/rules" -BEATS_VER="8.6.2" +BEATS_VER="8.7.1" BEATS_OSS="-oss" BEATS_DEB_URL_TEMPLATE_REPLACER="XXXXX" BEATS_DEB_URL_TEMPLATE="https://artifacts.elastic.co/downloads/beats/$BEATS_DEB_URL_TEMPLATE_REPLACER/$BEATS_DEB_URL_TEMPLATE_REPLACER$BEATS_OSS-$BEATS_VER-amd64.deb" diff --git a/sensor-iso/config/includes.chroot/etc/bash.bash_functions b/sensor-iso/config/includes.chroot/etc/bash.bash_functions index d555810cb..f53ac6309 100644 --- a/sensor-iso/config/includes.chroot/etc/bash.bash_functions +++ b/sensor-iso/config/includes.chroot/etc/bash.bash_functions @@ -223,6 +223,30 @@ function h() { if [ -z "$1" ]; then history; else history | grep -i "$@"; fi; } ######################################################################## function fname() { find . -iname "*$@*"; } +function findupes() { + find . -not -empty -type f -printf "%s\n" 2>/dev/null | \ + sort -rn | \ + uniq -d | \ + xargs -I{} -n1 find -type f -size {}c -print0 | \ + xargs -0 md5sum | \ + sort | \ + uniq -w32 --all-repeated=separate +} + +function sfind() { + if [ "$1" ]; then + FIND_FOLDER="$1" + else + FIND_FOLDER="$(pwd)" + fi + if [ "$2" ]; then + FIND_PATTERN="$2" + else + FIND_PATTERN="*" + fi + find "$FIND_FOLDER" -type f -iname "$FIND_PATTERN" -print0 | xargs -r -0 ls -la | awk '{system("numfmt -z --to=iec-i --suffix=B --padding=7 "$5) ; out=""; for(i=9;i<=NF;i++){out=out" "$i}; print " KB\t"out}' | sort -h +} + ######################################################################## # examine running processes ######################################################################## @@ -291,7 +315,36 @@ function arps() function portping() { - python <<<"import socket; socket.setdefaulttimeout(1); socket.socket().connect(('$1', $2))" 2> /dev/null && echo OPEN || echo CLOSED; + CONN_TIMEOUT=5 + if [[ -n "$BASH_VERSION" ]] && [[ $LINUX ]]; then + # use /dev/tcp + timeout $CONN_TIMEOUT bash -c "cat /dev/null > /dev/tcp/$1/$2" && echo OPEN || echo CLOSED + elif command -v python3 >/dev/null 2>&1; then + # use python socket library + python3 <<<"import socket; socket.setdefaulttimeout($CONN_TIMEOUT); socket.socket().connect(('$1', $2))" 2> /dev/null && echo OPEN || echo CLOSED + elif command -v socat >/dev/null 2>&1; then + # use socat + socat /dev/null TCP4:"$1":"$2",connect-timeout="$CONN_TIMEOUT" >/dev/null 2>&1 && echo OPEN || echo CLOSED + elif command -v nc >/dev/null 2>&1; then + # use some flavor of netcat + if ( nc -h 2>&1 | grep -q 'to somewhere' ); then + # traditional + ( timeout $((CONN_TIMEOUT+1)) bash -c "cat /dev/null | nc -v -w "$CONN_TIMEOUT" "$1" "$2" 2>&1" || true ) | grep -q 'open$' && echo OPEN || echo CLOSED + elif ( nc 2>&1 | grep -q '46CDdFhklNnrStUuvZz' ); then + # openbsd + timeout $((CONN_TIMEOUT+1)) bash -c "cat /dev/null | nc -w "$CONN_TIMEOUT" "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + elif ( nc --help 2>&1 | grep -q 'Ncat' ); then + # ncat + timeout $CONN_TIMEOUT bash -c "cat /dev/null | nc -v --send-only "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + else + echo UNKNOWN + fi + elif command -v telnet >/dev/null 2>&1; then + # use telnet + timeout $CONN_TIMEOUT bash -c "echo -e '\x1dclose\x0d' | telnet "$1" "$2" >/dev/null 2>&1" && echo OPEN || echo CLOSED + else + echo UNKNOWN + fi } ######################################################################## @@ -466,7 +519,12 @@ function sensormonitor () { select-pane -t 2 \; \ send-keys 'while true; do clear; /opt/sensor/sensor_ctl/status | grep -v "Not started" | sed "s/pid.* //"; sleep 60; done' C-m \; \ select-pane -t 3 \; \ - send-keys 'tail -F /opt/sensor/sensor_ctl/log/*' C-m + send-keys 'tail -F /opt/sensor/sensor_ctl/log/*' C-m \; \ + select-pane -t 2 \; \ + resize-pane -U 999 \; \ + resize-pane -D 27 \; \ + resize-pane -R 999 \; \ + resize-pane -L 58 } function suricata-update () { diff --git a/shared/bin/common-init.sh b/shared/bin/common-init.sh index 838cc52c9..ddc0bbd26 100755 --- a/shared/bin/common-init.sh +++ b/shared/bin/common-init.sh @@ -105,6 +105,7 @@ function FixPermissions() { echo "$USER_TO_FIX" >> /etc/at.allow fi chmod 644 /etc/cron.allow /etc/at.allow + loginctl enable-linger "$USER_TO_FIX" 2>/dev/null || true fi } diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index c02955e02..186b95538 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -25,7 +25,8 @@ function finish { rmdir -- "$LOCK_DIR" || echo "Failed to remove lock directory '$LOCK_DIR'" >&2 } -if mkdir -p -- "$LOCK_DIR" 2>/dev/null; then +mkdir -p -- "$(dirname "$LOCK_DIR")" +if mkdir -- "$LOCK_DIR" 2>/dev/null; then trap finish EXIT # create directive to @load every subdirectory in /opt/zeek/share/zeek/site/intel diff --git a/zeek/scripts/docker_entrypoint.sh b/zeek/scripts/docker_entrypoint.sh index 6fb0b975c..2d9d4f972 100755 --- a/zeek/scripts/docker_entrypoint.sh +++ b/zeek/scripts/docker_entrypoint.sh @@ -8,7 +8,7 @@ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/ze setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' "${ZEEK_DIR}"/bin/capstats || true if [[ "${ZEEK_LIVE_CAPTURE:-false}" != "true" ]] && [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then - sleep 5 # give the "live" instance, if there is one, a chance to go first + sleep 15 # give the "live" instance, if there is one, a chance to go first if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then su -s /bin/bash -p ${PUSER} << EOF "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true