Merge pull request #195 from idaholab/v23.05.1_merge_idaholab

Malcolm v23.05.1 is a minor release with a few component version updates and bug fixes, particularly to fix an issue with `install.py` where the ownership of `.env` files in the `config` directory may get incorrectly set to `root` rather than the unprivileged user. cisagov/Malcolm@v23.05.0...v23.05.1 * Enhancements and bug fixes - install.py can create .env files 0:0 ownership instead of unprivileged user ownership (cisagov#253) - both zeek and zeek-live containers are trying to pull intel feeds on startup (#196) - Make sure a few Arkime fields (`http.xff*`) get created in the index template with the right field types to avoid aggregation query issues - Tweaks to convenience scripts (`malcolmmonitor` and `sensormonitor`) in ISO-installed Malcolm and Hedgehog Linux environments - Added some `.service` files for the ISO-installed version of Malcolm to be able to feed itself resource statistics via Fluent Bit - Documentation updates * Component version updates - Arkime to [v4.3.1](https://github.com/arkime/arkime/blob/ce8d5d4d1a54a3a9f022bf4b72081f7af666f6e4/CHANGELOG#L33-L44) - OpenSearch and OpenSearch Dashboards to [v2.7.0](https://github.com/opensearch-project/opensearch-build/blob/2dbbce4428e583c4cf1f1f867f7591d978395420/release-notes/opensearch-release-notes-2.7.0.md) - NetBox to [v3.5.1](https://netbox.dev/blog/posts/netbox-v351-released/) - Beats to [v8.7.1](https://www.elastic.co/guide/en/beats/libbeat/8.7/release-notes-8.7.1.html)
idaholab · May 16, 2023 · 2a253f2 · 2a253f2
2 parents 8b73a54 + fbe668a
commit 2a253f2
Show file tree

Hide file tree

Showing 94 changed files with 1,065 additions and 408 deletions.
diff --git a/.github/workflows/api-build-and-push-ghcr.yml b/.github/workflows/api-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/arkime-build-and-push-ghcr.yml b/.github/workflows/arkime-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/dashboards-build-and-push-ghcr.yml b/.github/workflows/dashboards-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/dashboards-helper-build-and-push-ghcr.yml b/.github/workflows/dashboards-helper-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/file-monitor-build-and-push-ghcr.yml b/.github/workflows/file-monitor-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*.sh'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/file-upload-build-and-push-ghcr.yml b/.github/workflows/file-upload-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/filebeat-build-and-push-ghcr.yml b/.github/workflows/filebeat-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/freq-build-and-push-ghcr.yml b/.github/workflows/freq-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/htadmin-build-and-push-ghcr.yml b/.github/workflows/htadmin-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/logstash-build-and-push-ghcr.yml b/.github/workflows/logstash-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml b/.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml
@@ -9,6 +9,7 @@ on:
       - 'malcolm-iso/**'
       - 'shared/bin/*'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_iso_workflow_build'
       - '.github/workflows/malcolm-iso-build-docker-wrap-push-ghcr.yml'
   workflow_dispatch:

diff --git a/.github/workflows/netbox-build-and-push-ghcr.yml b/.github/workflows/netbox-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/nginx-build-and-push-ghcr.yml b/.github/workflows/nginx-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
       - '_config.yml'
       - '_includes/**'

diff --git a/.github/workflows/opensearch-build-and-push-ghcr.yml b/.github/workflows/opensearch-build-and-push-ghcr.yml
@@ -13,6 +13,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/pcap-capture-build-and-push-ghcr.yml b/.github/workflows/pcap-capture-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/pcap-monitor-build-and-push-ghcr.yml b/.github/workflows/pcap-monitor-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/postgresql-build-and-push-ghcr.yml b/.github/workflows/postgresql-build-and-push-ghcr.yml
@@ -13,6 +13,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/redis-build-and-push-ghcr.yml b/.github/workflows/redis-build-and-push-ghcr.yml
@@ -13,6 +13,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/.github/workflows/suricata-build-and-push-ghcr.yml b/.github/workflows/suricata-build-and-push-ghcr.yml
@@ -14,6 +14,7 @@ on:
       - '!shared/bin/sensor-init.sh'
       - '!shared/bin/configure-interfaces.py'
       - '!shared/bin/configure-capture.py'
+      - '!shared/bin/zeek*'
       - '.trigger_workflow_build'
   workflow_dispatch:
   repository_dispatch:

diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile
@@ -4,7 +4,7 @@ FROM debian:11-slim AS build
 
 ENV DEBIAN_FRONTEND noninteractive
 
-ENV ARKIME_VERSION "v4.3.0"
+ENV ARKIME_VERSION "v4.3.1"
 ENV ARKIME_DIR "/opt/arkime"
 ENV ARKIME_URL "https://github.com/arkime/arkime.git"
 ENV ARKIME_LOCALELASTICSEARCH no

diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile
@@ -14,10 +14,10 @@ ENV PGROUP "dashboarder"
 
 ENV TERM xterm
 
-ARG OPENSEARCH_VERSION="2.6.0"
+ARG OPENSEARCH_VERSION="2.7.0"
 ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION
 
-ARG OPENSEARCH_DASHBOARDS_VERSION="2.6.0"
+ARG OPENSEARCH_DASHBOARDS_VERSION="2.7.0"
 ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION
 
 # base system dependencies for checking out and building plugins
@@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \
 
 # runtime ##################################################################
 
-FROM opensearchproject/opensearch-dashboards:2.6.0
+FROM opensearchproject/opensearch-dashboards:2.7.0
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
@@ -90,7 +90,7 @@ ENV PUSER_PRIV_DROP true
 ENV TERM xterm
 
 ENV TINI_VERSION v0.19.0
-ENV OSD_TRANSFORM_VIS_VERSION 2.6.0
+ENV OSD_TRANSFORM_VIS_VERSION 2.7.0
 
 ARG OPENSEARCH_URL="http://opensearch:9200"
 ARG OPENSEARCH_LOCAL="true"
@@ -114,6 +114,7 @@ USER root
 
 COPY --from=build /usr/share/opensearch-dashboards/plugins/sankey_vis/build/kbnSankeyVis.zip /tmp/kbnSankeyVis.zip
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
+ADD https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip /tmp/transformVis.zip
 
 RUN yum upgrade -y && \
     yum install -y curl psmisc util-linux openssl rsync python3 zip unzip && \
@@ -122,7 +123,14 @@ RUN yum upgrade -y && \
     /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \
     cd /usr/share/opensearch-dashboards/plugins && \
     /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \
-    /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \
+    cd /tmp && \
+        # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
+        # sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \
+        # sed -i "s/2\.6\.0/2\.7\.0/g" opensearch-dashboards/transformVis/package.json && \
+        # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \
+        cd /usr/share/opensearch-dashboards/plugins && \
+        /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \
+        rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \
     chown --silent -R root:root /usr/share/opensearch-dashboards/plugins/* \
                                 /usr/share/opensearch-dashboards/node_modules/* \
                                 /usr/share/opensearch-dashboards/src/* && \

diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:8.6.2
+FROM docker.elastic.co/beats/filebeat-oss:8.7.1
 
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile
@@ -73,7 +73,8 @@ RUN apt-get -q update && \
       mv /etc/unit/nginx-unit-new.json /etc/unit/nginx-unit.json && \
       chmod 644 /etc/unit/nginx-unit.json && \
     tr -cd '\11\12\15\40-\176' < /opt/netbox/netbox/netbox/configuration.py > /opt/netbox/netbox/netbox/configuration_ascii.py && \
-      mv /opt/netbox/netbox/netbox/configuration_ascii.py /opt/netbox/netbox/netbox/configuration.py
+      mv /opt/netbox/netbox/netbox/configuration_ascii.py /opt/netbox/netbox/netbox/configuration.py && \
+    sed -i -E 's@^([[:space:]]*\-\-(state|tmp))([[:space:]])@\1dir\3@g' /opt/netbox/launch-netbox.sh
 
 COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 COPY --chmod=755 shared/bin/service_check_passthrough.sh /usr/local/bin/

diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile
@@ -1,4 +1,4 @@
-FROM opensearchproject/opensearch:2.6.0
+FROM opensearchproject/opensearch:2.7.0
 
 # Copyright (c) 2023 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

diff --git a/config/logstash.env.example b/config/logstash.env.example
@@ -12,4 +12,4 @@ LOGSTASH_REVERSE_DNS=false
 # Whether or not Logstash will enrich network traffic metadata via NetBox API calls
 LOGSTASH_NETBOX_ENRICHMENT=false
 # Logstash memory allowance and other Java options
-LS_JAVA_OPTS=-server -Xms2g -Xmx2g -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true
+LS_JAVA_OPTS=-server -Xms2500m -Xmx2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true
diff --git a/config/opensearch.env.example b/config/opensearch.env.example
@@ -37,7 +37,7 @@ OPENSEARCH_SECONDARY_CREDS_CONFIG_FILE=/var/local/curlrc/.opensearch.secondary.c
 #  certificates).
 OPENSEARCH_SECONDARY_SSL_CERTIFICATE_VERIFICATION=false
 # OpenSearch memory allowance and other Java options
-OPENSEARCH_JAVA_OPTS=-server -Xms4g -Xmx4g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true
+OPENSEARCH_JAVA_OPTS=-server -Xms10g -Xmx10g -Xss256k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true
 
 logger.level=WARN
 bootstrap.memory_lock=true

diff --git a/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json b/dashboards/dashboards/03207c00-d07e-11ec-b4a7-d1b4003706b7.json
@@ -12,7 +12,7 @@
       "attributes": {
         "title": "GENISYS",
         "hits": 0,
-        "description": "Dashboard for the DNP3 Protocol",
+        "description": "Dashboard for the GENISYS Protocol",
         "panelsJSON": "[{\"version\":\"1.3.1\",\"gridData\":{\"h\":28,\"i\":\"1\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"1\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":10,\"i\":\"58856fb7-efd0-4246-9dc9-d8b0d5c3fcba\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"58856fb7-efd0-4246-9dc9-d8b0d5c3fcba\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":10,\"i\":\"c078d6a7-456e-4fed-80c6-f36123c3ba82\",\"w\":32,\"x\":16,\"y\":0},\"panelIndex\":\"c078d6a7-456e-4fed-80c6-f36123c3ba82\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"c04b22a5-6b7e-4c18-8172-d39ec8549e4a\",\"w\":8,\"x\":8,\"y\":10},\"panelIndex\":\"c04b22a5-6b7e-4c18-8172-d39ec8549e4a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"4da40cc7-ad85-4dd1-88cf-8b207995c932\",\"w\":12,\"x\":16,\"y\":10},\"panelIndex\":\"4da40cc7-ad85-4dd1-88cf-8b207995c932\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_4\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"74347ef4-7a00-4d8f-a172-120339fd5e30\",\"w\":20,\"x\":28,\"y\":10},\"panelIndex\":\"74347ef4-7a00-4d8f-a172-120339fd5e30\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"40ffbd38-1edc-4493-b313-6f65729cbe70\",\"w\":16,\"x\":0,\"y\":28},\"panelIndex\":\"40ffbd38-1edc-4493-b313-6f65729cbe70\",\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"panelRefName\":\"panel_6\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"2cb13858-f268-4cd4-8207-3932c70dc83a\",\"w\":12,\"x\":16,\"y\":28},\"panelIndex\":\"2cb13858-f268-4cd4-8207-3932c70dc83a\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}},\"table\":null},\"panelRefName\":\"panel_7\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":18,\"i\":\"7aabaf8b-4a54-48df-ac8e-c732327f420e\",\"w\":20,\"x\":28,\"y\":28},\"panelIndex\":\"7aabaf8b-4a54-48df-ac8e-c732327f420e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"},{\"version\":\"1.3.1\",\"gridData\":{\"h\":31,\"i\":\"6b987e44-72f1-4e33-9fa3-cb21c7313829\",\"w\":48,\"x\":0,\"y\":46},\"panelIndex\":\"6b987e44-72f1-4e33-9fa3-cb21c7313829\",\"embeddableConfig\":{},\"panelRefName\":\"panel_9\"}]",
         "optionsJSON": "{\"useMargins\":true}",
         "version": 1,

diff --git a/dashboards/templates/composable/component/arkime.json b/dashboards/templates/composable/component/arkime.json
@@ -11,6 +11,10 @@
         "destination.geo.longitude": { "type": "float" },
         "dns.host": { "type": "keyword" },
         "firstPacket": { "type": "date" },
+        "http.xffASN": { "type": "keyword" },
+        "http.xffGEO": { "type": "keyword" },
+        "http.xffIp": { "type": "ip" },
+        "http.xffRIR": { "type": "keyword" },
         "lastPacket": { "type": "date" },
         "node": { "type": "keyword" },
         "protocol": { "type": "keyword" },