From bf3da933278e3726b47dc846e1203e55fe52450d Mon Sep 17 00:00:00 2001
From: SG <13872653+mmguero@users.noreply.github.com>
Date: Fri, 3 Sep 2021 13:38:43 -0600
Subject: [PATCH] 3.3.0 development (#173)
* New features
* Automatically create some broadly useful anomaly detectors when initializing Kibana
* connection size
* file transfer MIME type
* action and result (by application protocol)
* Configurable [event severity scoring](https://github.com/cisagov/malcolm/tree/main#Severity) (idaholab/Malcolm#19) and new **Severity** dashboard
* Other changes
* vagrant-based ISO build can now work with either VirtualBox or libvirt providers
* change wording of terms such as "master"/"slave" to "client"/"server" as instructed by DHS directive
* Version updates
* Update base image for Debian-based Docker images from 10 (buster) to 11 (bullseye)
* Update Yara to 4.1.2
* Update Capa to 2.0.0
* Update Spicy to 1.2.1
* Update remainder of python 2 code to python 3
---
Dockerfiles/arkime.Dockerfile | 15 +-
Dockerfiles/elasticsearch.Dockerfile | 4 +-
Dockerfiles/file-monitor.Dockerfile | 17 +-
Dockerfiles/file-upload.Dockerfile | 19 +-
Dockerfiles/filebeat.Dockerfile | 4 +-
Dockerfiles/freq.Dockerfile | 11 +-
Dockerfiles/htadmin.Dockerfile | 11 +-
Dockerfiles/kibana-helper.Dockerfile | 7 +-
Dockerfiles/kibana.Dockerfile | 8 +-
Dockerfiles/logstash.Dockerfile | 4 +-
Dockerfiles/name-map-ui.Dockerfile | 4 +-
Dockerfiles/nginx.Dockerfile | 7 +-
Dockerfiles/pcap-capture.Dockerfile | 10 +-
Dockerfiles/pcap-monitor.Dockerfile | 8 +-
Dockerfiles/zeek.Dockerfile | 43 +-
Notice.txt | 4 +-
README.md | 140 +-
docker-compose-standalone.yml | 50 +-
docker-compose.yml | 50 +-
docs/contributing/README.md | 57 +
file-upload/nginx/sites-available/default | 2 +-
file-upload/supervisord.conf | 2 +-
htadmin/htadmin.sh | 2 +-
htadmin/nginx/sites-available/default | 2 +-
htadmin/supervisord.conf | 2 +-
kibana/anomaly_detectors/total_bytes_AD.json | 47 +
.../anomaly_detectors/zeek_filetype_AD.json | 47 +
kibana/anomaly_detectors/zeek_service_AD.json | 73 +
.../024062a6-48d6-498f-a91a-3bf2da3a3cd3.json | 2 +-
.../05e3e000-f118-11e9-acda-83a8e29e1a24.json | 2 +-
.../078b9aa5-9bd4-4f02-ae5e-cf80fa6f887b.json | 2 +-
.../0a490422-0ce9-44bf-9a2d-19329ddde8c3.json | 2 +-
.../0ad3d7c2-3441-485e-9dfe-dbb22e84e576.json | 2 +-
.../0aed0e23-c8ac-4f2b-9f68-d04b6e7666b0.json | 2 +-
.../0b2354ae-0fe9-4fd9-b156-1c3870e5c7aa.json | 2 +-
.../11be6381-beef-40a7-bdce-88c5398392fc.json | 2 +-
.../11ddd980-e388-11e9-b568-cf17de8e860c.json | 2 +-
.../12e3a130-d83b-11eb-a0b0-f328ce09b0b7.json | 2 +-
.../152f29dc-51a2-4f53-93e9-6e92765567b8.json | 16 +-
.../1ce42250-3f99-11e9-a58e-8bdedb0915e8.json | 2 +-
.../1fff49f6-0199-4a0f-820b-721aff9ff1f1.json | 2 +-
.../29a1b290-eb98-11e9-a384-0fcf32210194.json | 2 +-
.../2bec1490-eb94-11e9-a384-0fcf32210194.json | 2 +-
.../2cf94cd0-ecab-40a5-95a7-8419f3a39cd9.json | 2 +-
.../2d98bb8e-214c-4374-837b-20e1bcd63a5e.json | 2 +-
.../32587740-ef88-11e9-b38a-2db3ee640e88.json | 2 +-
.../36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json | 2 +-
.../37041ee1-79c0-4684-a436-3173b0e89876.json | 2 +-
.../39abfe30-3f99-11e9-a58e-8bdedb0915e8.json | 2 +-
.../42e831b9-41a9-4f35-8b7d-e1566d368773.json | 2 +-
.../432af556-c5c0-4cc3-8166-b274b4e3a406.json | 2 +-
.../4a073440-b286-11eb-a4d4-09fa12a6ebd4.json | 4 +-
.../4a4bde20-4760-11ea-949c-bbb5a9feecbf.json | 2 +-
.../4e5f106e-c60a-4226-8f64-d534abb912ab.json | 2 +-
.../50ced171-1b10-4c3f-8b67-2db9635661a6.json | 2 +-
.../543118a9-02d7-43fe-b669-b8652177fc37.json | 2 +-
.../55e332d0-3f99-11e9-a58e-8bdedb0915e8.json | 2 +-
.../60d78fbd-471c-4f59-a9e3-189b33a13644.json | 2 +-
.../665d1610-523d-11e9-a30e-e3576242f3ed.json | 2 +-
.../76f2f912-80da-44cd-ab66-6a73c8344cc3.json | 2 +-
.../77fc9960-3f99-11e9-a58e-8bdedb0915e8.json | 2 +-
.../7f41913f-cba8-43f5-82a8-241b7ead03e0.json | 2 +-
.../7f77b58a-df3e-4cc2-b782-fd7f8bad8ffb.json | 1486 ++++++++---------
.../82da3101-2a9c-4ae2-bb61-d447a3fbe673.json | 2 +-
.../870a5862-6c26-4a08-99fd-0c06cda85ba3.json | 2 +-
.../87a32f90-ef58-11e9-974e-9d600036d105.json | 2 +-
.../87d990cc-9e0b-41e5-b8fe-b10ae1da0c85.json | 2 +-
.../92985909-dc29-4533-9e80-d3182a0ecf1d.json | 2 +-
.../95479950-41f2-11ea-88fa-7151df485405.json | 2 +-
.../9ee51f94-3316-4fc5-bd89-93a52af69714.json | 2 +-
.../a16110b0-3f99-11e9-a58e-8bdedb0915e8.json | 2 +-
.../a33e0a50-afcd-11ea-993f-b7d8522a8bed.json | 2 +-
.../a7514350-eba6-11e9-a384-0fcf32210194.json | 2 +-
.../abdd7550-2c7c-40dc-947e-f6d186a158c4.json | 2 +-
.../ae79b7d1-4281-4095-b2f6-fa7eafda9970.json | 2 +-
.../af5df620-eeb6-11e9-bdef-65a192b7f586.json | 2 +-
.../b50c8d17-6ed3-4de6-aed4-5181032810b2.json | 2 +-
.../b9f247c0-3f99-11e9-a58e-8bdedb0915e8.json | 2 +-
.../bb827f8e-639e-468c-93c8-9f5bc132eb8f.json | 2 +-
.../bed185a0-ef82-11e9-b38a-2db3ee640e88.json | 2 +-
.../bf5efbb0-60f1-11eb-9d60-dbf0411cfc48.json | 2 +-
.../c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2.json | 2 +-
.../ca5799a0-56b5-11eb-b749-576de068f8ad.json | 2 +-
.../caef3ade-d289-4d05-a511-149f3e97f238.json | 2 +-
.../d2dd0180-06b1-11ec-8c6b-353266ade330.json | 687 ++++++++
.../d41fe630-3f98-11e9-a58e-8bdedb0915e8.json | 2 +-
.../d4fd6afd-15cb-42bf-8a25-03dd8e59b327.json | 2 +-
.../e09a4b86-29b5-4256-bb3b-802ac9f90404.json | 2 +-
.../e76d05c0-eb9f-11e9-a384-0fcf32210194.json | 2 +-
.../ed8a6640-3f98-11e9-a58e-8bdedb0915e8.json | 2 +-
.../f1f09567-fc7f-450b-a341-19d2f2bb468b.json | 1015 +++++------
.../f394057d-1b16-4174-b994-7045f423a416.json | 2 +-
.../f77bf097-18a8-465c-b634-eb2acc7a4f26.json | 2 +-
.../fa141950-ef89-11e9-b38a-2db3ee640e88.json | 2 +-
.../kibana-create-moloch-sessions-index.sh | 13 +
kibana/zeek_template.json | 9 +-
logstash/maps/malcolm_severity.yaml | 32 +
logstash/maps/s7comm_result_codes.yaml | 2 +-
logstash/maps/service_ports.yaml | 218 +++
.../pipelines/enrichment/19_severity.conf | 454 +++++
.../enrichment/20_enriched_to_ecs.conf | 4 +-
logstash/pipelines/zeek/11_zeek_logs.conf | 59 +-
.../pipelines/zeek/12_zeek_normalize.conf | 15 +
logstash/pipelines/zeek/20_zeek_to_ecs.conf | 17 +-
malcolm-iso/build.sh | 2 +
malcolm-iso/vagrant/Vagrantfile | 12 +-
moloch/etc/config.ini | 19 +-
moloch/wise/source.zeeklogs.js | 21 +-
pcap-capture/scripts/netsniff-roll.sh | 2 +-
pcap-capture/scripts/supervisor.sh | 2 +-
pcap-capture/templates/tcpdump.template | 2 +-
scripts/beats/README.md | 6 +-
..._Windows_Forwarder_Download_and_Config.ps1 | 8 +-
scripts/build.sh | 2 +-
scripts/malcolm_appliance_packager.sh | 2 +
sensor-iso/README.md | 21 +-
sensor-iso/beats/Dockerfile | 2 +-
sensor-iso/build.sh | 2 +-
.../normal/0910-sensor-build.hook.chroot | 6 +-
.../usr/local/etc/zeek/guess_ics_map.txt | 6 +-
.../usr/local/etc/zeek/local.zeek | 15 +-
.../interface/sensor_ctl/control_vars.conf | 4 +
sensor-iso/moloch/Dockerfile | 2 +-
sensor-iso/vagrant/Vagrantfile | 12 +-
shared/bin/malass_client.py | 3 +-
shared/bin/zeek_carve_utils.py | 2 +-
zeek/config/guess_ics_map.txt | 6 +-
zeek/config/local.zeek | 15 +-
128 files changed, 3412 insertions(+), 1565 deletions(-)
create mode 100644 docs/contributing/README.md
create mode 100644 kibana/anomaly_detectors/total_bytes_AD.json
create mode 100644 kibana/anomaly_detectors/zeek_filetype_AD.json
create mode 100644 kibana/anomaly_detectors/zeek_service_AD.json
create mode 100644 kibana/dashboards/d2dd0180-06b1-11ec-8c6b-353266ade330.json
create mode 100644 logstash/maps/malcolm_severity.yaml
create mode 100644 logstash/maps/service_ports.yaml
create mode 100644 logstash/pipelines/enrichment/19_severity.conf
diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile
index 459bcac24..25c6cbfe7 100644
--- a/Dockerfiles/arkime.Dockerfile
+++ b/Dockerfiles/arkime.Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:buster-slim AS build
+FROM debian:bullseye-slim AS build
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
@@ -16,8 +16,7 @@ ADD README.md $ARKIMEDIR/doc/
ADD doc.css $ARKIMEDIR/doc/
ADD docs/images $ARKIMEDIR/doc/images/
-RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
- apt-get -q update && \
+RUN apt-get -q update && \
apt-get install -q -y --no-install-recommends \
binutils \
bison \
@@ -84,10 +83,10 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
npm cache clean --force && \
bash -c "file ${ARKIMEDIR}/bin/* ${ARKIMEDIR}/node-v*/bin/* | grep 'ELF 64-bit' | sed 's/:.*//' | xargs -l -r strip -v --strip-unneeded"
-FROM debian:buster-slim
+FROM debian:bullseye-slim
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -142,7 +141,7 @@ ENV PCAP_MONITOR_HOST $PCAP_MONITOR_HOST
COPY --from=build $ARKIMEDIR $ARKIMEDIR
-RUN sed -i "s/buster main/buster main contrib non-free/" /etc/apt/sources.list && \
+RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.list && \
apt-get -q update && \
apt-get install -q -y --no-install-recommends \
curl \
@@ -176,7 +175,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/" /etc/apt/sources.list &
ln -sfr $ARKIMEDIR/bin/npm /usr/local/bin/npm && \
ln -sfr $ARKIMEDIR/bin/node /usr/local/bin/node && \
ln -sfr $ARKIMEDIR/bin/npx /usr/local/bin/npx && \
- apt-get -q -y --purge remove gcc gcc-8 cpp cpp-8 libssl-dev && \
+ apt-get -q -y --purge remove gcc gcc-10 cpp cpp-10 libssl-dev && \
apt-get -q -y autoremove && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
diff --git a/Dockerfiles/elasticsearch.Dockerfile b/Dockerfiles/elasticsearch.Dockerfile
index 553a70b5a..a2847fef7 100644
--- a/Dockerfiles/elasticsearch.Dockerfile
+++ b/Dockerfiles/elasticsearch.Dockerfile
@@ -1,8 +1,8 @@
FROM amazon/opendistro-for-elasticsearch:1.13.2
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile
index ce76201f8..0dbedcca7 100644
--- a/Dockerfiles/file-monitor.Dockerfile
+++ b/Dockerfiles/file-monitor.Dockerfile
@@ -1,8 +1,8 @@
-FROM debian:buster-slim
+FROM debian:bullseye-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -81,12 +81,12 @@ ENV EXTRACTED_FILE_ENABLE_CAPA $EXTRACTED_FILE_ENABLE_CAPA
ENV EXTRACTED_FILE_CAPA_VERBOSE $EXTRACTED_FILE_CAPA_VERBOSE
ENV SRC_BASE_DIR "/usr/local/src"
ENV CLAMAV_RULES_DIR "/var/lib/clamav"
-ENV YARA_VERSION "4.1.1"
+ENV YARA_VERSION "4.1.2"
ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
ENV YARA_RULES_URL "https://github.com/Neo23x0/signature-base"
ENV YARA_RULES_DIR "/yara-rules"
ENV YARA_RULES_SRC_DIR "$SRC_BASE_DIR/signature-base"
-ENV CAPA_VERSION "1.6.3"
+ENV CAPA_VERSION "2.0.0"
ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip"
ENV CAPA_DIR "/opt/capa"
ENV CAPA_BIN "${CAPA_DIR}/capa"
@@ -102,8 +102,8 @@ ENV SUPERCRONIC "supercronic-linux-amd64"
ENV SUPERCRONIC_SHA1SUM "048b95b48b708983effb2e5c935a1ef8483d9e3e"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
-RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
- apt-get update && \
+RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.list && \
+ apt-get -q update && \
apt-get install --no-install-recommends -y -q \
automake \
bc \
@@ -183,9 +183,6 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/* && \
mkdir -p /var/log/clamav "${CLAMAV_RULES_DIR}" && \
- curl -s -S -L -o "${CLAMAV_RULES_DIR}"/main.cvd http://database.clamav.net/main.cvd && \
- curl -s -S -L -o "${CLAMAV_RULES_DIR}"/daily.cvd http://database.clamav.net/daily.cvd && \
- curl -s -S -L -o "${CLAMAV_RULES_DIR}"/bytecode.cvd http://database.clamav.net/bytecode.cvd && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
useradd -m --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} && \
usermod -a -G tty ${PUSER} && \
diff --git a/Dockerfiles/file-upload.Dockerfile b/Dockerfiles/file-upload.Dockerfile
index a57b128a5..2a9d3eb66 100644
--- a/Dockerfiles/file-upload.Dockerfile
+++ b/Dockerfiles/file-upload.Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:buster-slim AS build
+FROM debian:bullseye-slim AS build
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
@@ -23,10 +23,10 @@ RUN apt-get update && \
rm -rf /jQuery-File-Upload/*.html /jQuery-File-Upload/test/ /jQuery-File-Upload/server/gae-go/ \
/jQuery-File-Upload/server/gae-python/
-FROM debian:buster-slim AS runtime
+FROM debian:bullseye-slim AS runtime
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -48,6 +48,9 @@ ENV PUSER_PRIV_DROP false
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
+ARG PHP_VERSION=7.4
+ENV PHP_VERSION $PHP_VERSION
+
COPY --from=build /jQuery-File-Upload/ /var/www/upload/
RUN apt-get update && \
@@ -58,9 +61,9 @@ RUN apt-get update && \
supervisor \
vim-tiny \
less \
- php7.3-gd \
- php7.3-fpm \
- php7.3-apcu \
+ php$PHP_VERSION \
+ php$PHP_VERSION-fpm \
+ php$PHP_VERSION-apcu \
nginx-light && \
apt-get clean -y -q && \
rm -rf /var/lib/apt/lists/*
@@ -72,7 +75,7 @@ ADD file-upload/jquery-file-upload/bootstrap.min.css /var/www/upload/bower_compo
ADD file-upload/jquery-file-upload/index.html /var/www/upload/index.html
ADD file-upload/jquery-file-upload/index.php /var/www/upload/server/php/index.php
ADD file-upload/nginx/sites-available/default /etc/nginx/sites-available/default
-ADD file-upload/php/php.ini /etc/php/7.3/fpm/php.ini
+ADD file-upload/php/php.ini /etc/php/$PHP_VERSION/fpm/php.ini
ADD file-upload/sshd_config /tmp/sshd_config
ADD file-upload/supervisord.conf /supervisord.conf
diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
index d368c0900..aa0543a1a 100644
--- a/Dockerfiles/filebeat.Dockerfile
+++ b/Dockerfiles/filebeat.Dockerfile
@@ -1,8 +1,8 @@
FROM docker.elastic.co/beats/filebeat-oss:7.10.2
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
diff --git a/Dockerfiles/freq.Dockerfile b/Dockerfiles/freq.Dockerfile
index 0fa628bec..480e2cfa2 100644
--- a/Dockerfiles/freq.Dockerfile
+++ b/Dockerfiles/freq.Dockerfile
@@ -1,8 +1,8 @@
-FROM debian:buster-slim
+FROM debian:bullseye-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -29,8 +29,7 @@ ENV FREQ_LOOKUP $FREQ_LOOKUP
ENV FREQ_URL "https://codeload.github.com/markbaggett/freq/tar.gz/master"
-RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list && \
- apt-get update && \
+RUN apt-get update && \
apt-get -y -q install \
curl \
procps \
@@ -38,7 +37,7 @@ RUN sed -i "s/buster main/buster main contrib non-free/g" /etc/apt/sources.list
python3 \
python3-dev \
python3-pip && \
- pip3 install supervisor && \
+ pip3 install supervisor six && \
cd /opt && \
mkdir -p ./freq_server && \
curl -sSL "$FREQ_URL" | tar xzvf - -C ./freq_server --strip-components 1 && \
diff --git a/Dockerfiles/htadmin.Dockerfile b/Dockerfiles/htadmin.Dockerfile
index 29831a943..4e870f1a3 100644
--- a/Dockerfiles/htadmin.Dockerfile
+++ b/Dockerfiles/htadmin.Dockerfile
@@ -1,8 +1,8 @@
-FROM debian:buster-slim
+FROM debian:bullseye-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -23,8 +23,8 @@ ENV PUSER_PRIV_DROP false
ENV DEBIAN_FRONTEND noninteractive
ENV TERM xterm
-ARG PHP_VERSION=7.3
-ARG MCRYPT_VERSION=1.0.2
+ARG PHP_VERSION=7.4
+ARG MCRYPT_VERSION=1.0.4
ARG BOOTSTRAP_VERSION=3.3.6
ENV PHP_VERSION $PHP_VERSION
@@ -35,7 +35,6 @@ ENV HTADMIN_URL "https://codeload.github.com/mmguero-dev/htadmin/tar.gz/master"
RUN apt-get update && \
apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --no-install-recommends install \
- bcrypt \
ca-certificates \
curl \
libmcrypt-dev \
diff --git a/Dockerfiles/kibana-helper.Dockerfile b/Dockerfiles/kibana-helper.Dockerfile
index 355e906b6..32950f835 100644
--- a/Dockerfiles/kibana-helper.Dockerfile
+++ b/Dockerfiles/kibana-helper.Dockerfile
@@ -1,8 +1,8 @@
FROM alpine:3.14
# Copyright (c) 2020 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -48,6 +48,7 @@ ENV SUPERCRONIC_SHA1SUM "048b95b48b708983effb2e5c935a1ef8483d9e3e"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
ADD kibana/dashboards /opt/kibana/dashboards
+ADD kibana/anomaly_detectors /opt/kibana/anomaly_detectors
ADD kibana/maps /opt/maps
ADD kibana/scripts /data/
ADD kibana/supervisord.conf /etc/supervisord.conf
@@ -69,7 +70,7 @@ RUN apk --no-cache add bash python3 py3-pip curl procps psmisc npm shadow jq &&
addgroup ${PUSER} tty ; \
addgroup ${PUSER} shadow ; \
mkdir -p /data/init && \
- chown -R ${PUSER}:${PGROUP} /opt/kibana/dashboards /opt/maps /data/init && \
+ chown -R ${PUSER}:${PGROUP} /opt/kibana/dashboards /opt/maps /data/init /opt/kibana/anomaly_detectors && \
chmod 755 /data/*.sh /data/*.py /data/init && \
chmod 400 /opt/maps/* && \
(echo -e "*/2 * * * * /data/kibana-create-moloch-sessions-index.sh\n0 10 * * * /data/kibana_index_refresh.py --template zeek_template\n*/20 * * * * /data/elastic_index_size_prune.py" > ${SUPERCRONIC_CRONTAB})
diff --git a/Dockerfiles/kibana.Dockerfile b/Dockerfiles/kibana.Dockerfile
index a37d231b9..209fdfbe8 100644
--- a/Dockerfiles/kibana.Dockerfile
+++ b/Dockerfiles/kibana.Dockerfile
@@ -54,7 +54,7 @@ RUN git clone --single-branch --depth=1 --recurse-submodules --shallow-submodule
RUN eval "$(nodenv init -)" && \
mkdir -p /usr/share/kibana/plugins && \
- git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch --branch feature/update_7.10.1 https://github.com/mmguero-dev/kbn_sankey_vis.git /usr/share/kibana/plugins/sankey_vis && \
+ git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch --branch master https://github.com/mmguero-dev/kbn_sankey_vis.git /usr/share/kibana/plugins/sankey_vis && \
cd /usr/share/kibana/plugins/sankey_vis && \
yarn kbn bootstrap && \
yarn install && \
@@ -63,8 +63,8 @@ RUN eval "$(nodenv init -)" && \
FROM amazon/opendistro-for-elasticsearch-kibana:1.13.2
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -88,6 +88,7 @@ ARG ARKIME_INDEX_PATTERN="sessions2-*"
ARG ARKIME_INDEX_PATTERN_ID="sessions2-*"
ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
ARG KIBANA_DEFAULT_DASHBOARD="0ad3d7c2-3441-485e-9dfe-dbb22e84e576"
+ARG NODE_OPTIONS="--max_old_space_size=4096"
ENV CREATE_ES_ARKIME_SESSION_INDEX $CREATE_ES_ARKIME_SESSION_INDEX
ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
@@ -99,6 +100,7 @@ ENV KIBANA_OFFLINE_REGION_MAPS_PORT $KIBANA_OFFLINE_REGION_MAPS_PORT
ENV PATH="/data:${PATH}"
ENV ELASTICSEARCH_URL $ELASTICSEARCH_URL
ENV KIBANA_DEFAULT_DASHBOARD $KIBANA_DEFAULT_DASHBOARD
+ENV NODE_OPTIONS $NODE_OPTIONS
USER root
diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile
index 0632acd0a..a6194a459 100644
--- a/Dockerfiles/logstash.Dockerfile
+++ b/Dockerfiles/logstash.Dockerfile
@@ -43,8 +43,8 @@ RUN cd /opt && \
FROM docker.elastic.co/logstash/logstash-oss:7.10.2
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
diff --git a/Dockerfiles/name-map-ui.Dockerfile b/Dockerfiles/name-map-ui.Dockerfile
index 753369a11..f5700684e 100644
--- a/Dockerfiles/name-map-ui.Dockerfile
+++ b/Dockerfiles/name-map-ui.Dockerfile
@@ -1,8 +1,8 @@
FROM alpine:3.14
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
diff --git a/Dockerfiles/nginx.Dockerfile b/Dockerfiles/nginx.Dockerfile
index 59baf013e..1e178708c 100644
--- a/Dockerfiles/nginx.Dockerfile
+++ b/Dockerfiles/nginx.Dockerfile
@@ -34,16 +34,17 @@ RUN set -x ; \
cd /apkbuild ; \
tar xvf /aports-master.tar.gz aports-master/community/stunnel ; \
cd /apkbuild/aports-master/community/stunnel ; \
+ sed -i "s@www\.stunnel\.org/downloads/@ftp.osuosl.org/pub/blfs/conglomeration/stunnel/@" APKBUILD ; \
abuild-keygen -a -i -n ; \
abuild checksum ; \
- abuild -R
+ abuild -f -R
####################################################################################
FROM alpine:3.13
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
diff --git a/Dockerfiles/pcap-capture.Dockerfile b/Dockerfiles/pcap-capture.Dockerfile
index c4fe28306..315026b01 100644
--- a/Dockerfiles/pcap-capture.Dockerfile
+++ b/Dockerfiles/pcap-capture.Dockerfile
@@ -1,8 +1,8 @@
-FROM debian:buster-slim
+FROM debian:bullseye-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -74,8 +74,8 @@ RUN apt-get update && \
chmod -R 750 /etc/supervisor.d && \
chown root:${PGROUP} /sbin/ethtool && \
setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \
- chown root:${PGROUP} /usr/sbin/tcpdump && \
- setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/sbin/tcpdump && \
+ chown root:${PGROUP} /usr/bin/tcpdump && \
+ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /usr/bin/tcpdump && \
chown root:${PGROUP} /usr/sbin/netsniff-ng && \
setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip CAP_SYS_ADMIN+eip' /usr/sbin/netsniff-ng && \
chmod 755 /usr/local/bin/*.sh
diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile
index c21a99d17..94ae44f18 100644
--- a/Dockerfiles/pcap-monitor.Dockerfile
+++ b/Dockerfiles/pcap-monitor.Dockerfile
@@ -1,8 +1,8 @@
-FROM debian:buster-slim
+FROM debian:bullseye-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -53,7 +53,7 @@ RUN apt-get update && \
vim-tiny && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
- pip3 install --no-cache-dir elasticsearch elasticsearch_dsl pyzmq pyinotify python-magic && \
+ pip3 install --no-cache-dir elasticsearch==7.10.1 elasticsearch_dsl==7.4.0 pyzmq pyinotify python-magic && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER}
diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
index 6ccbcbab3..e36a9e500 100644
--- a/Dockerfiles/zeek.Dockerfile
+++ b/Dockerfiles/zeek.Dockerfile
@@ -1,9 +1,9 @@
-FROM debian:buster-slim
+FROM debian:bullseye-slim
# Copyright (c) 2021 Battelle Energy Alliance, LLC. All rights reserved.
-LABEL maintainer="malcolm.netsec@gmail.com"
-LABEL org.opencontainers.image.authors='malcolm.netsec@gmail.com'
+LABEL maintainer="malcolm@inl.gov"
+LABEL org.opencontainers.image.authors='malcolm@inl.gov'
LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/main/README.md'
LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
@@ -26,7 +26,7 @@ ENV PUSER_PRIV_DROP true
# for download and install
ARG ZEEK_LTS=1
ARG ZEEK_VERSION=4.0.3-0
-ARG SPICY_VERSION=1.1.0
+ARG SPICY_VERSION=1.2.1
ENV ZEEK_LTS $ZEEK_LTS
ENV ZEEK_VERSION $ZEEK_VERSION
@@ -49,8 +49,7 @@ ENV PATH "${ZEEK_DIR}/bin:${SPICY_DIR}/bin:${ZEEK_DIR}/lib/zeek/plugins/packages
ADD shared/bin/zeek_install_plugins.sh /usr/local/bin/
# build and install system packages, zeek, spicy and plugins
-RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list && \
- apt-get -q update && \
+RUN apt-get -q update && \
apt-get install -q -y --no-install-recommends \
ca-certificates \
curl \
@@ -65,10 +64,10 @@ RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/so
psmisc \
vim-tiny && \
( curl -sSL https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - ) && \
- echo "deb http://apt.llvm.org/buster/ llvm-toolchain-buster-${LLVM_VERSION} main" >> /etc/apt/sources.list && \
- echo "deb-src http://apt.llvm.org/buster/ llvm-toolchain-buster-${LLVM_VERSION} main" >> /etc/apt/sources.list && \
+ echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${LLVM_VERSION} main" >> /etc/apt/sources.list && \
+ echo "deb-src http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${LLVM_VERSION} main" >> /etc/apt/sources.list && \
apt-get -q update && \
- apt-get install -q -y -t buster-backports --no-install-recommends \
+ apt-get install -q -y --no-install-recommends \
bison \
ccache \
clang-${LLVM_VERSION} \
@@ -104,14 +103,14 @@ RUN echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/so
cd /tmp/zeek-packages && \
if [ -n "${ZEEK_LTS}" ]; then ZEEK_LTS="-lts"; fi && export ZEEK_LTS && \
curl -sSL --remote-name-all \
- "https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/libbroker${ZEEK_LTS}-dev_${ZEEK_VERSION}_amd64.deb" \
- "https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-core-dev_${ZEEK_VERSION}_amd64.deb" \
- "https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-core_${ZEEK_VERSION}_amd64.deb" \
- "https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-libcaf-dev_${ZEEK_VERSION}_amd64.deb" \
- "https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}_${ZEEK_VERSION}_amd64.deb" \
- "https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-btest_${ZEEK_VERSION}_amd64.deb" \
- "https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeek${ZEEK_LTS}-zkg_${ZEEK_VERSION}_amd64.deb" \
- "https://download.opensuse.org/repositories/security:/zeek/Debian_10/amd64/zeekctl${ZEEK_LTS}_${ZEEK_VERSION}_amd64.deb" && \
+ "https://download.opensuse.org/repositories/security:/zeek/Debian_11/amd64/libbroker${ZEEK_LTS}-dev_${ZEEK_VERSION}_amd64.deb" \
+ "https://download.opensuse.org/repositories/security:/zeek/Debian_11/amd64/zeek${ZEEK_LTS}-core-dev_${ZEEK_VERSION}_amd64.deb" \
+ "https://download.opensuse.org/repositories/security:/zeek/Debian_11/amd64/zeek${ZEEK_LTS}-core_${ZEEK_VERSION}_amd64.deb" \
+ "https://download.opensuse.org/repositories/security:/zeek/Debian_11/amd64/zeek${ZEEK_LTS}-libcaf-dev_${ZEEK_VERSION}_amd64.deb" \
+ "https://download.opensuse.org/repositories/security:/zeek/Debian_11/amd64/zeek${ZEEK_LTS}_${ZEEK_VERSION}_amd64.deb" \
+ "https://download.opensuse.org/repositories/security:/zeek/Debian_11/amd64/zeek${ZEEK_LTS}-btest_${ZEEK_VERSION}_amd64.deb" \
+ "https://download.opensuse.org/repositories/security:/zeek/Debian_11/amd64/zeek${ZEEK_LTS}-zkg_${ZEEK_VERSION}_amd64.deb" \
+ "https://download.opensuse.org/repositories/security:/zeek/Debian_11/amd64/zeekctl${ZEEK_LTS}_${ZEEK_VERSION}_amd64.deb" && \
dpkg -i ./*.deb && \
mkdir -p /tmp/spicy-packages && \
cd /tmp/spicy-packages && \
@@ -201,9 +200,13 @@ ARG ZEEK_DISABLE_BEST_GUESS_ICS=true
# for now, disable them by default when a Zeek parser exists
ARG ZEEK_DISABLE_SPICY_DHCP=true
ARG ZEEK_DISABLE_SPICY_DNS=true
+ARG ZEEK_DISABLE_SPICY_FACEFISH=
ARG ZEEK_DISABLE_SPICY_HTTP=true
ARG ZEEK_DISABLE_SPICY_IPSEC=
+ARG ZEEK_DISABLE_SPICY_LDAP=
ARG ZEEK_DISABLE_SPICY_OPENVPN=
+ARG ZEEK_DISABLE_SPICY_STUN=
+ARG ZEEK_DISABLE_SPICY_TAILSCALE=
ARG ZEEK_DISABLE_SPICY_TFTP=
ARG ZEEK_DISABLE_SPICY_WIREGUARD=
@@ -212,10 +215,16 @@ ENV ZEEK_DISABLE_LOG_PASSWORDS $ZEEK_DISABLE_LOG_PASSWORDS
ENV ZEEK_DISABLE_SSL_VALIDATE_CERTS $ZEEK_DISABLE_SSL_VALIDATE_CERTS
ENV ZEEK_DISABLE_TRACK_ALL_ASSETS $ZEEK_DISABLE_TRACK_ALL_ASSETS
ENV ZEEK_DISABLE_BEST_GUESS_ICS $ZEEK_DISABLE_BEST_GUESS_ICS
+
ENV ZEEK_DISABLE_SPICY_DHCP $ZEEK_DISABLE_SPICY_DHCP
ENV ZEEK_DISABLE_SPICY_DNS $ZEEK_DISABLE_SPICY_DNS
+ENV ZEEK_DISABLE_SPICY_FACEFISH $ZEEK_DISABLE_SPICY_FACEFISH
ENV ZEEK_DISABLE_SPICY_HTTP $ZEEK_DISABLE_SPICY_HTTP
+ENV ZEEK_DISABLE_SPICY_IPSEC $ZEEK_DISABLE_SPICY_IPSEC
+ENV ZEEK_DISABLE_SPICY_LDAP $ZEEK_DISABLE_SPICY_LDAP
ENV ZEEK_DISABLE_SPICY_OPENVPN $ZEEK_DISABLE_SPICY_OPENVPN
+ENV ZEEK_DISABLE_SPICY_STUN $ZEEK_DISABLE_SPICY_STUN
+ENV ZEEK_DISABLE_SPICY_TAILSCALE $ZEEK_DISABLE_SPICY_TAILSCALE
ENV ZEEK_DISABLE_SPICY_TFTP $ZEEK_DISABLE_SPICY_TFTP
ENV ZEEK_DISABLE_SPICY_WIREGUARD $ZEEK_DISABLE_SPICY_WIREGUARD
diff --git a/Notice.txt b/Notice.txt
index 8ffd0cae8..a52feb52a 100644
--- a/Notice.txt
+++ b/Notice.txt
@@ -34,6 +34,4 @@ COMPLETENESS, OR USEFULNESS OF ANY DATA, APPARATUS, PRODUCT, OR PROCESS DISCLOSE
OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS.
-Authors:
-
-Seth Grover (malcolm.netsec@gmail.com)
+Contact the authors: malcolm@inl.gov
diff --git a/README.md b/README.md
index 51a3e7371..b4f5106cd 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,8 @@ In short, Malcolm provides an easily deployable network analysis tool suite for
+ [Defining hostname and CIDR subnet names interface](#NameMapUI)
+ [Applying mapping changes](#ApplyMapping)
- [Elasticsearch index management](#IndexManagement)
+ - [Event severity scoring](#Severity)
+ + [Customizing event severity scoring](#SeverityConfig)
- [Alerting](#Alerting)
- ["Best Guess" Fingerprinting for ICS Protocols](#ICSBestGuess)
* [Using Beats to forward host logs to Malcolm](#OtherBeats)
@@ -139,21 +141,21 @@ You can then observe that the images have been retrieved by running `docker imag
```
$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
-malcolmnetsec/arkime 3.2.1 xxxxxxxxxxxx 39 hours ago 683MB
-malcolmnetsec/elasticsearch-od 3.2.1 xxxxxxxxxxxx 40 hours ago 690MB
-malcolmnetsec/file-monitor 3.2.1 xxxxxxxxxxxx 39 hours ago 470MB
-malcolmnetsec/file-upload 3.2.1 xxxxxxxxxxxx 39 hours ago 199MB
-malcolmnetsec/filebeat-oss 3.2.1 xxxxxxxxxxxx 39 hours ago 555MB
-malcolmnetsec/freq 3.2.1 xxxxxxxxxxxx 39 hours ago 390MB
-malcolmnetsec/htadmin 3.2.1 xxxxxxxxxxxx 39 hours ago 180MB
-malcolmnetsec/kibana-helper 3.2.1 xxxxxxxxxxxx 40 hours ago 141MB
-malcolmnetsec/kibana-od 3.2.1 xxxxxxxxxxxx 40 hours ago 1.16GB
-malcolmnetsec/logstash-oss 3.2.1 xxxxxxxxxxxx 39 hours ago 1.41GB
-malcolmnetsec/name-map-ui 3.2.1 xxxxxxxxxxxx 39 hours ago 137MB
-malcolmnetsec/nginx-proxy 3.2.1 xxxxxxxxxxxx 39 hours ago 120MB
-malcolmnetsec/pcap-capture 3.2.1 xxxxxxxxxxxx 39 hours ago 111MB
-malcolmnetsec/pcap-monitor 3.2.1 xxxxxxxxxxxx 39 hours ago 157MB
-malcolmnetsec/zeek 3.2.1 xxxxxxxxxxxx 39 hours ago 887MB
+malcolmnetsec/arkime 3.3.0 xxxxxxxxxxxx 39 hours ago 683MB
+malcolmnetsec/elasticsearch-od 3.3.0 xxxxxxxxxxxx 40 hours ago 690MB
+malcolmnetsec/file-monitor 3.3.0 xxxxxxxxxxxx 39 hours ago 470MB
+malcolmnetsec/file-upload 3.3.0 xxxxxxxxxxxx 39 hours ago 199MB
+malcolmnetsec/filebeat-oss 3.3.0 xxxxxxxxxxxx 39 hours ago 555MB
+malcolmnetsec/freq 3.3.0 xxxxxxxxxxxx 39 hours ago 390MB
+malcolmnetsec/htadmin 3.3.0 xxxxxxxxxxxx 39 hours ago 180MB
+malcolmnetsec/kibana-helper 3.3.0 xxxxxxxxxxxx 40 hours ago 141MB
+malcolmnetsec/kibana-od 3.3.0 xxxxxxxxxxxx 40 hours ago 1.16GB
+malcolmnetsec/logstash-oss 3.3.0 xxxxxxxxxxxx 39 hours ago 1.41GB
+malcolmnetsec/name-map-ui 3.3.0 xxxxxxxxxxxx 39 hours ago 137MB
+malcolmnetsec/nginx-proxy 3.3.0 xxxxxxxxxxxx 39 hours ago 120MB
+malcolmnetsec/pcap-capture 3.3.0 xxxxxxxxxxxx 39 hours ago 111MB
+malcolmnetsec/pcap-monitor 3.3.0 xxxxxxxxxxxx 39 hours ago 157MB
+malcolmnetsec/zeek 3.3.0 xxxxxxxxxxxx 39 hours ago 887MB
```
#### Import from pre-packaged tarballs
@@ -485,6 +487,18 @@ Various other environment variables inside of `docker-compose.yml` can be tweake
* `LOGSTASH_REVERSE_DNS` – if set to `true`, Logstash will perform a reverse DNS lookup for all external source and destination IP address values when analyzing Zeek logs (default `false`)
+* `LOGSTASH_SEVERITY_SCORING` - if set to `true`, Logstash will perform [severity scoring](#Severity) when analyzing Zeek logs (default `true`)
+
+* `FREQ_LOOKUP` - if set to `true`, domain names (from DNS queries and SSL server names) will be assigned entropy scores as calculated by [`freq`](https://github.com/MarkBaggett/freq) (default `false`)
+
+* `FREQ_SEVERITY_THRESHOLD` - when [severity scoring](#Severity) is enabled, this variable indicates the entropy threshold for assigning severity to events with entropy scores calculated by [`freq`](https://github.com/MarkBaggett/freq); a lower value will only assign severity scores to fewer domain names with higher entropy (e.g., `2.0` for `NQZHTFHRMYMTVBQJE.COM`), while a higher value will assign severity scores to more domain names with lower entropy (e.g., `7.5` for `naturallanguagedomain.example.org`) (default `2.0`)
+
+* `TOTAL_MEGABYTES_SEVERITY_THRESHOLD` - when [severity scoring](#Severity) is enabled, this variable indicates the size threshold (in megabytes) for assigning severity to large connections or file transfers (default `1000`)
+
+* `CONNECTION_SECONDS_SEVERITY_THRESHOLD` - when [severity scoring](#Severity) is enabled, this variable indicates the duration threshold (in seconds) for assigning severity to long connections (default `3600`)
+
+* `QUESTIONABLE_COUNTRY_CODES` - when [severity scoring](#Severity) is enabled, this variable defines a comma-separated list of countries of concern (using [ISO 3166-1 alpha-2 codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Current_codes)) (default `'CN,IR,KP,RU,UA'`)
+
* `ES_EXTERNAL_HOSTS` – if specified (in the format `'10.0.0.123:9200'`), logs received by Logstash will be forwarded on to another external Elasticsearch instance in addition to the one maintained locally by Malcolm
* `ES_EXTERNAL_SSL` – if set to `true`, Logstash will use HTTPS for the connection to external Elasticsearch instances specified in `ES_EXTERNAL_HOSTS`
@@ -780,7 +794,7 @@ Authentication over LDAP can be done using one of three ways, [two of which](htt
* **StartTLS** - the [standard extension](https://tools.ietf.org/html/rfc2830) to the LDAP protocol to establish an encrypted SSL/TLS connection within an already established LDAP connection
* **LDAPS** - a commonly used (though unofficial and considered deprecated) method in which SSL negotiation takes place before any commands are sent from the client to the server
-* **Unencrypted** (clear text) (***not recommended***)
+* **Unencrypted** (cleartext) (***not recommended***)
In addition to the `NGINX_BASIC_AUTH` environment variable being set to `false` in the `x-auth-variables` section near the top of the [`docker-compose.yml`](#DockerComposeYml) file, the `NGINX_LDAP_TLS_STUNNEL` and `NGINX_LDAP_TLS_STUNNEL` environment variables are used in conjunction with the values in `nginx/nginx_ldap.conf` to define the LDAP connection security level. Use the following combinations of values to achieve the connection security methods above, respectively:
@@ -1373,6 +1387,49 @@ See [Index State Management](https://opendistro.github.io/for-elasticsearch-docs
Elasticsearch index management only deals with disk space consumed by Elasticsearch indices: it does not have anything to do with PCAP file storage. The `MANAGE_PCAP_FILES` environment variable in the [`docker-compose.yml`](#DockerComposeYml) file can be used to allow Arkime to prune old PCAP files based on available disk space.
+### Event severity scoring
+
+As Zeek logs are parsed and enriched prior to indexing, a severity score up to `100` (a higher score indicating a more severe event) can be assigned when one or more of the following conditions are met:
+
+* cross-segment network traffic (if [network subnets were defined](#HostAndSubnetNaming))
+* connection origination and destination (e.g., inbound, outbound, external, internal)
+* traffic to or from countries of concern
+ - The comma-separated list of countries (by [ISO 3166-1 alpha-2 code](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Current_codes)) can be customized by setting the `QUESTIONABLE_COUNTRY_CODES` environment variable in [`docker-compose.yml`](#DockerComposeYml).
+* domain names (from DNS queries and SSL server names) with high entropy as calculated by [freq](https://github.com/MarkBaggett/freq)
+ - The entropy threshold for this condition to trigger can be adjusted by setting the `FREQ_SEVERITY_THRESHOLD` environment variable in [`docker-compose.yml`](#DockerComposeYml). A lower value will only assign severity scores to fewer domain names with higher entropy (e.g., `2.0` for `NQZHTFHRMYMTVBQJE.COM`), while a higher value will assign severity scores to more domain names with lower entropy (e.g., `7.5` for `naturallanguagedomain.example.org`).
+* file transfers (categorized by mime type)
+* `notice.log` and `weird.log` entries, including those generated by Zeek plugins detecting vulnerabilities (see the list of Zeek plugins under [Components](#Components))
+* detection of cleartext passwords
+* use of insecure or outdated protocols
+* tunneled traffic or use of VPN protocols
+* rejected or aborted connections
+* common network services communicating over non-standard ports
+* file scanning engine hits on [extracted files](#ZeekFileExtraction)
+* large connection or file transfer
+ - The size (in megabytes) threshold for this condition to trigger can be adjusted by setting the `TOTAL_MEGABYTES_SEVERITY_THRESHOLD` environment variable in [`docker-compose.yml`](#DockerComposeYml).
+* long connection duration
+ - The duration (in seconds) threshold for this condition to trigger can be adjusted by setting the `CONNECTION_SECONDS_SEVERITY_THRESHOLD` environment variable in [`docker-compose.yml`](#DockerComposeYml).
+
+As this [feature](https://github.com/idaholab/Malcolm/issues/19) is improved it's expected that additional categories will be identified and implemented for severity scoring.
+
+When a Zeek log satisfies more than one of these conditions its severity scores will be summed, with a maximum score of `100`. A Zeek log's severity score is indexed in the `event.severity` field and the conditions which contributed to its score are indexed in `event.severity_tags`.
+
+#### Customizing event severity scoring
+
+These categories' severity scores can be customized by editing `logstash/maps/malcolm_severity.yaml`:
+
+* Each category can be assigned a number between `1` and `100` for severity scoring.
+* Any category may be disabled by assigning it a score of `0`.
+* A severity score can be assigned for any [supported protocol](#Protocols) by adding an entry with the key formatted like `"PROTOCOL_XYZ"`, where `XYZ` is the uppercased value of the protocol as stored in the `zeek.service` field. For example, to assign a score of `40` to Zeek logs generated for SSH traffic, you could add the following line to `malcolm_severity.yaml`:
+
+```
+"PROTOCOL_SSH": 40
+```
+
+Restart Logstash after modifying `malcolm_severity.yaml` for the changes to take effect. The [hostname and CIDR subnet names interface](#NameMapUI) provides a convenient button for restarting Logstash.
+
+Severity scoring can be disabled globally by setting the `LOGSTASH_SEVERITY_SCORING` environment variable to `false` in the [`docker-compose.yml`](#DockerComposeYml) file and [restarting Malcolm](#StopAndRestart).
+
### Alerting
See [Alerting](https://opendistro.github.io/for-elasticsearch-docs/docs/alerting/) in the Open Distro for Elasticsearch documentation.
@@ -1406,7 +1463,7 @@ This action should only be performed while Malcolm is [stopped](#StopAndRestart)
There are many ICS (industrial control systems) protocols. While Malcolm's collection of [protocol parsers](#Protocols) includes a number of them, many, particularly those that are proprietary or less common, are unlikely to be supported with a full parser in the foreseeable future.
-In an effort to help identify more ICS traffic, Malcolm can use "buest guess" method based on transport protocol (e.g., TCP or UDP) and port(s) to categorize potential traffic communicating over some ICS protocols without full parser support. This feature involves a [mapping table](https://github.com/idaholab/Malcolm/blob/master/zeek/config/guess_ics_map.txt) and a [Zeek script](https://github.com/idaholab/Malcolm/blob/master/zeek/config/guess.zeek) to look up the transport protocol and destination and/or source port to make a best guess at whether a connection belongs to one of those protocols. These potential ICS communications are categorized by vendor where possible.
+In an effort to help identify more ICS traffic, Malcolm can use "buest guess" method based on transport protocol (e.g., TCP or UDP) and port(s) to categorize potential traffic communicating over some ICS protocols without full parser support. This feature involves a [mapping table](https://github.com/cisagov/Malcolm/blob/master/zeek/config/guess_ics_map.txt) and a [Zeek script](https://github.com/cisagov/Malcolm/blob/master/zeek/config/guess.zeek) to look up the transport protocol and destination and/or source port to make a best guess at whether a connection belongs to one of those protocols. These potential ICS communications are categorized by vendor where possible.
Naturally, these lookups could produce false positives, so these connections are displayed in their own dashboard (the **Best Guess** dashboard found under the **ICS** section of Malcolm's [Kibana dashboards'](#KibanaVisualizations) navigation pane). Values such as IP addresses, ports, or UID can be used to [pivot to other dashboards](#ZeekArkimeFlowCorrelation) to investigate further.
@@ -1424,7 +1481,20 @@ Malcolm can be packaged into an installer ISO based on the current [stable relea
### Generating the ISO
-Official downloads of the Malcolm installer ISO are not provided: however, it can be built easily on an internet-connected Linux host running current versions of [VirtualBox](https://www.virtualbox.org/) and [Vagrant](https://www.vagrantup.com/) (with the [`vagrant-reload`](https://github.com/aidanns/vagrant-reload) plugin).
+Official downloads of the Malcolm installer ISO are not provided: however, it can be built easily on an internet-connected Linux host with Vagrant:
+
+* [Vagrant](https://www.vagrantup.com/)
+ - [`vagrant-reload`](https://github.com/aidanns/vagrant-reload) plugin
+ - [`vagrant-sshfs`](https://github.com/dustymabe/vagrant-sshfs) plugin
+ - [`bento/debian-10`](https://app.vagrantup.com/bento/boxes/debian-10) Vagrant box
+
+The build should work with either the [VirtualBox](https://www.virtualbox.org/) provider or the [libvirt](https://libvirt.org/) provider:
+
+* [VirtualBox](https://www.virtualbox.org/) [provider](https://www.vagrantup.com/docs/providers/virtualbox)
+ - [`vagrant-vbguest`](https://github.com/dotless-de/vagrant-vbguest) plugin
+* [libvirt](https://libvirt.org/)
+ - [`vagrant-libvirt`](https://github.com/vagrant-libvirt/vagrant-libvirt) provider plugin
+ - [`vagrant-mutate`](https://github.com/sciurus/vagrant-mutate) plugin to convert [`bento/debian-10`](https://app.vagrantup.com/bento/boxes/debian-10) Vagrant box to `libvirt` format
To perform a clean build the Malcolm installer ISO, navigate to your local Malcolm working copy and run:
@@ -1440,7 +1510,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu
```
…
-Finished, created "/malcolm-build/malcolm-iso/malcolm-3.2.1.iso"
+Finished, created "/malcolm-build/malcolm-iso/malcolm-3.3.0.iso"
…
```
@@ -1823,21 +1893,21 @@ Pulling zeek ... done
user@host:~/Malcolm$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
-malcolmnetsec/arkime 3.2.1 xxxxxxxxxxxx 39 hours ago 683MB
-malcolmnetsec/elasticsearch-od 3.2.1 xxxxxxxxxxxx 40 hours ago 690MB
-malcolmnetsec/file-monitor 3.2.1 xxxxxxxxxxxx 39 hours ago 470MB
-malcolmnetsec/file-upload 3.2.1 xxxxxxxxxxxx 39 hours ago 199MB
-malcolmnetsec/filebeat-oss 3.2.1 xxxxxxxxxxxx 39 hours ago 555MB
-malcolmnetsec/freq 3.2.1 xxxxxxxxxxxx 39 hours ago 390MB
-malcolmnetsec/htadmin 3.2.1 xxxxxxxxxxxx 39 hours ago 180MB
-malcolmnetsec/kibana-helper 3.2.1 xxxxxxxxxxxx 40 hours ago 141MB
-malcolmnetsec/kibana-od 3.2.1 xxxxxxxxxxxx 40 hours ago 1.16GB
-malcolmnetsec/logstash-oss 3.2.1 xxxxxxxxxxxx 39 hours ago 1.41GB
-malcolmnetsec/name-map-ui 3.2.1 xxxxxxxxxxxx 39 hours ago 137MB
-malcolmnetsec/nginx-proxy 3.2.1 xxxxxxxxxxxx 39 hours ago 120MB
-malcolmnetsec/pcap-capture 3.2.1 xxxxxxxxxxxx 39 hours ago 111MB
-malcolmnetsec/pcap-monitor 3.2.1 xxxxxxxxxxxx 39 hours ago 157MB
-malcolmnetsec/zeek 3.2.1 xxxxxxxxxxxx 39 hours ago 887MB
+malcolmnetsec/arkime 3.3.0 xxxxxxxxxxxx 39 hours ago 683MB
+malcolmnetsec/elasticsearch-od 3.3.0 xxxxxxxxxxxx 40 hours ago 690MB
+malcolmnetsec/file-monitor 3.3.0 xxxxxxxxxxxx 39 hours ago 470MB
+malcolmnetsec/file-upload 3.3.0 xxxxxxxxxxxx 39 hours ago 199MB
+malcolmnetsec/filebeat-oss 3.3.0 xxxxxxxxxxxx 39 hours ago 555MB
+malcolmnetsec/freq 3.3.0 xxxxxxxxxxxx 39 hours ago 390MB
+malcolmnetsec/htadmin 3.3.0 xxxxxxxxxxxx 39 hours ago 180MB
+malcolmnetsec/kibana-helper 3.3.0 xxxxxxxxxxxx 40 hours ago 141MB
+malcolmnetsec/kibana-od 3.3.0 xxxxxxxxxxxx 40 hours ago 1.16GB
+malcolmnetsec/logstash-oss 3.3.0 xxxxxxxxxxxx 39 hours ago 1.41GB
+malcolmnetsec/name-map-ui 3.3.0 xxxxxxxxxxxx 39 hours ago 137MB
+malcolmnetsec/nginx-proxy 3.3.0 xxxxxxxxxxxx 39 hours ago 120MB
+malcolmnetsec/pcap-capture 3.3.0 xxxxxxxxxxxx 39 hours ago 111MB
+malcolmnetsec/pcap-monitor 3.3.0 xxxxxxxxxxxx 39 hours ago 157MB
+malcolmnetsec/zeek 3.3.0 xxxxxxxxxxxx 39 hours ago 887MB
```
Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background.
@@ -1962,6 +2032,6 @@ See [`License.txt`](./License.txt) for the terms of its release.
## Contact information of author(s):
-[Seth Grover](mailto:malcolm.netsec@gmail.com?subject=Malcolm)
+[malcolm@inl.gov](mailto:malcolm@inl.gov?subject=Malcolm)
[![Join the chat at https://gitter.im/malcolmnetsec/community](https://badges.gitter.im/malcolmnetsec/community.svg)](https://gitter.im/malcolmnetsec/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml
index 7d837eec8..6e62c7b3d 100644
--- a/docker-compose-standalone.yml
+++ b/docker-compose-standalone.yml
@@ -67,9 +67,13 @@ x-zeek-variables: &zeek-variables
ZEEK_DISABLE_BEST_GUESS_ICS : 'true'
ZEEK_DISABLE_SPICY_DHCP : 'true'
ZEEK_DISABLE_SPICY_DNS : 'true'
+ ZEEK_DISABLE_SPICY_FACEFISH : ''
ZEEK_DISABLE_SPICY_HTTP : 'true'
ZEEK_DISABLE_SPICY_IPSEC : ''
+ ZEEK_DISABLE_SPICY_LDAP : ''
ZEEK_DISABLE_SPICY_OPENVPN : ''
+ ZEEK_DISABLE_SPICY_STUN : ''
+ ZEEK_DISABLE_SPICY_TAILSCALE : ''
ZEEK_DISABLE_SPICY_TFTP : ''
ZEEK_DISABLE_SPICY_WIREGUARD : ''
@@ -85,8 +89,9 @@ x-kibana-helper-variables: &kibana-helper-variables
ISM_SNAPSHOT_REPO : 'logs'
x-logstash-variables: &logstash-variables
- LOGSTASH_OUI_LOOKUP : 'true'
- LOGSTASH_REVERSE_DNS : 'false'
+ LOGSTASH_OUI_LOOKUP : 'true'
+ LOGSTASH_SEVERITY_SCORING : 'true'
+ LOGSTASH_REVERSE_DNS : 'false'
# ES_EXTERNAL_HOSTS : '10.0.0.123:9200'
# ES_EXTERNAL_SSL : 'true'
# ES_EXTERNAL_SSL_CERTIFICATE_VERIFICATION : 'false'
@@ -104,6 +109,10 @@ x-common-upload-variables: &common-upload-variables
x-common-lookup-variables: &common-lookup-variables
FREQ_LOOKUP : 'false'
+ FREQ_SEVERITY_THRESHOLD : '2.0'
+ TOTAL_MEGABYTES_SEVERITY_THRESHOLD : 1000
+ CONNECTION_SECONDS_SEVERITY_THRESHOLD : 3600
+ QUESTIONABLE_COUNTRY_CODES : 'CN,IR,KP,RU,UA'
x-common-beats-variables: &common-beats-variables
BEATS_SSL : 'false'
@@ -119,7 +128,7 @@ x-pcap-capture-variables: &pcap-capture-variables
services:
elasticsearch:
- image: malcolmnetsec/elasticsearch-od:3.2.1
+ image: malcolmnetsec/elasticsearch-od:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -156,7 +165,7 @@ services:
retries: 3
start_period: 180s
kibana-helper:
- image: malcolmnetsec/kibana-helper:3.2.1
+ image: malcolmnetsec/kibana-helper:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -184,7 +193,7 @@ services:
retries: 3
start_period: 30s
kibana:
- image: malcolmnetsec/kibana-od:3.2.1
+ image: malcolmnetsec/kibana-od:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -205,7 +214,7 @@ services:
retries: 3
start_period: 210s
logstash:
- image: malcolmnetsec/logstash-oss:3.2.1
+ image: malcolmnetsec/logstash-oss:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -225,6 +234,7 @@ services:
- 9600
volumes:
- ./logstash/certs/logstash.keystore:/usr/share/logstash/config/logstash.keystore:rw
+ - ./logstash/maps/malcolm_severity.yaml:/etc/malcolm_severity.yaml:ro
- ./nginx/ca-trust:/usr/share/logstash/ca-trust:ro
- ./logstash/certs/ca.crt:/certs/ca.crt:ro
- ./logstash/certs/server.crt:/certs/server.crt:ro
@@ -239,7 +249,7 @@ services:
retries: 3
start_period: 600s
filebeat:
- image: malcolmnetsec/filebeat-oss:3.2.1
+ image: malcolmnetsec/filebeat-oss:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -276,7 +286,7 @@ services:
retries: 3
start_period: 60s
arkime:
- image: malcolmnetsec/arkime:3.2.1
+ image: malcolmnetsec/arkime:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -315,7 +325,7 @@ services:
retries: 3
start_period: 210s
zeek:
- image: malcolmnetsec/zeek:3.2.1
+ image: malcolmnetsec/zeek:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -341,7 +351,7 @@ services:
retries: 3
start_period: 60s
file-monitor:
- image: malcolmnetsec/file-monitor:3.2.1
+ image: malcolmnetsec/file-monitor:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -364,7 +374,7 @@ services:
retries: 3
start_period: 60s
pcap-capture:
- image: malcolmnetsec/pcap-capture:3.2.1
+ image: malcolmnetsec/pcap-capture:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -383,14 +393,8 @@ services:
<< : *pcap-capture-variables
volumes:
- ./pcap/upload:/pcap
- healthcheck:
- test: ["CMD", "supervisorctl", "status"]
- interval: 30s
- timeout: 15s
- retries: 3
- start_period: 60s
pcap-monitor:
- image: malcolmnetsec/pcap-monitor:3.2.1
+ image: malcolmnetsec/pcap-monitor:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -413,7 +417,7 @@ services:
retries: 3
start_period: 90s
upload:
- image: malcolmnetsec/file-upload:3.2.1
+ image: malcolmnetsec/file-upload:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -439,7 +443,7 @@ services:
retries: 3
start_period: 60s
htadmin:
- image: malcolmnetsec/htadmin:3.2.1
+ image: malcolmnetsec/htadmin:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -461,7 +465,7 @@ services:
retries: 3
start_period: 60s
freq:
- image: malcolmnetsec/freq:3.2.1
+ image: malcolmnetsec/freq:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -479,7 +483,7 @@ services:
retries: 3
start_period: 60s
name-map-ui:
- image: malcolmnetsec/name-map-ui:3.2.1
+ image: malcolmnetsec/name-map-ui:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -500,7 +504,7 @@ services:
retries: 3
start_period: 60s
nginx-proxy:
- image: malcolmnetsec/nginx-proxy:3.2.1
+ image: malcolmnetsec/nginx-proxy:3.3.0
restart: "no"
stdin_open: false
tty: true
diff --git a/docker-compose.yml b/docker-compose.yml
index cd1ebf833..d9e892f8d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -67,9 +67,13 @@ x-zeek-variables: &zeek-variables
ZEEK_DISABLE_BEST_GUESS_ICS : 'true'
ZEEK_DISABLE_SPICY_DHCP : 'true'
ZEEK_DISABLE_SPICY_DNS : 'true'
+ ZEEK_DISABLE_SPICY_FACEFISH : ''
ZEEK_DISABLE_SPICY_HTTP : 'true'
ZEEK_DISABLE_SPICY_IPSEC : ''
+ ZEEK_DISABLE_SPICY_LDAP : ''
ZEEK_DISABLE_SPICY_OPENVPN : ''
+ ZEEK_DISABLE_SPICY_STUN : ''
+ ZEEK_DISABLE_SPICY_TAILSCALE : ''
ZEEK_DISABLE_SPICY_TFTP : ''
ZEEK_DISABLE_SPICY_WIREGUARD : ''
@@ -85,8 +89,9 @@ x-kibana-helper-variables: &kibana-helper-variables
ISM_SNAPSHOT_REPO : 'logs'
x-logstash-variables: &logstash-variables
- LOGSTASH_OUI_LOOKUP : 'true'
- LOGSTASH_REVERSE_DNS : 'false'
+ LOGSTASH_OUI_LOOKUP : 'true'
+ LOGSTASH_SEVERITY_SCORING : 'true'
+ LOGSTASH_REVERSE_DNS : 'false'
# ES_EXTERNAL_HOSTS : '10.0.0.123:9200'
# ES_EXTERNAL_SSL : 'true'
# ES_EXTERNAL_SSL_CERTIFICATE_VERIFICATION : 'false'
@@ -104,6 +109,10 @@ x-common-upload-variables: &common-upload-variables
x-common-lookup-variables: &common-lookup-variables
FREQ_LOOKUP : 'false'
+ FREQ_SEVERITY_THRESHOLD : '2.0'
+ TOTAL_MEGABYTES_SEVERITY_THRESHOLD : 1000
+ CONNECTION_SECONDS_SEVERITY_THRESHOLD : 3600
+ QUESTIONABLE_COUNTRY_CODES : 'CN,IR,KP,RU,UA'
x-common-beats-variables: &common-beats-variables
BEATS_SSL : 'false'
@@ -122,7 +131,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/elasticsearch.Dockerfile
- image: malcolmnetsec/elasticsearch-od:3.2.1
+ image: malcolmnetsec/elasticsearch-od:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -162,7 +171,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/kibana-helper.Dockerfile
- image: malcolmnetsec/kibana-helper:3.2.1
+ image: malcolmnetsec/kibana-helper:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -193,7 +202,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/kibana.Dockerfile
- image: malcolmnetsec/kibana-od:3.2.1
+ image: malcolmnetsec/kibana-od:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -217,7 +226,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/logstash.Dockerfile
- image: malcolmnetsec/logstash-oss:3.2.1
+ image: malcolmnetsec/logstash-oss:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -238,6 +247,7 @@ services:
volumes:
- ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
- ./logstash/pipelines:/usr/share/logstash/malcolm-pipelines.available:ro
+ - ./logstash/maps/malcolm_severity.yaml:/etc/malcolm_severity.yaml:ro
- ./logstash/certs/logstash.keystore:/usr/share/logstash/config/logstash.keystore:rw
- ./nginx/ca-trust:/usr/share/logstash/ca-trust:ro
- ./logstash/certs/ca.crt:/certs/ca.crt:ro
@@ -256,7 +266,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/filebeat.Dockerfile
- image: malcolmnetsec/filebeat-oss:3.2.1
+ image: malcolmnetsec/filebeat-oss:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -297,7 +307,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/arkime.Dockerfile
- image: malcolmnetsec/arkime:3.2.1
+ image: malcolmnetsec/arkime:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -342,7 +352,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/zeek.Dockerfile
- image: malcolmnetsec/zeek:3.2.1
+ image: malcolmnetsec/zeek:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -372,7 +382,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/file-monitor.Dockerfile
- image: malcolmnetsec/file-monitor:3.2.1
+ image: malcolmnetsec/file-monitor:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -398,7 +408,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/pcap-capture.Dockerfile
- image: malcolmnetsec/pcap-capture:3.2.1
+ image: malcolmnetsec/pcap-capture:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -417,17 +427,11 @@ services:
<< : *pcap-capture-variables
volumes:
- ./pcap/upload:/pcap
- healthcheck:
- test: ["CMD", "supervisorctl", "status"]
- interval: 30s
- timeout: 15s
- retries: 3
- start_period: 60s
pcap-monitor:
build:
context: .
dockerfile: Dockerfiles/pcap-monitor.Dockerfile
- image: malcolmnetsec/pcap-monitor:3.2.1
+ image: malcolmnetsec/pcap-monitor:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -453,7 +457,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/file-upload.Dockerfile
- image: malcolmnetsec/file-upload:3.2.1
+ image: malcolmnetsec/file-upload:3.3.0
restart: "no"
stdin_open: false
tty: true
@@ -479,7 +483,7 @@ services:
retries: 3
start_period: 60s
htadmin:
- image: malcolmnetsec/htadmin:3.2.1
+ image: malcolmnetsec/htadmin:3.3.0
build:
context: .
dockerfile: Dockerfiles/htadmin.Dockerfile
@@ -504,7 +508,7 @@ services:
retries: 3
start_period: 60s
freq:
- image: malcolmnetsec/freq:3.2.1
+ image: malcolmnetsec/freq:3.3.0
build:
context: .
dockerfile: Dockerfiles/freq.Dockerfile
@@ -525,7 +529,7 @@ services:
retries: 3
start_period: 60s
name-map-ui:
- image: malcolmnetsec/name-map-ui:3.2.1
+ image: malcolmnetsec/name-map-ui:3.3.0
build:
context: .
dockerfile: Dockerfiles/name-map-ui.Dockerfile
@@ -552,7 +556,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/nginx.Dockerfile
- image: malcolmnetsec/nginx-proxy:3.2.1
+ image: malcolmnetsec/nginx-proxy:3.3.0
restart: "no"
stdin_open: false
tty: true
diff --git a/docs/contributing/README.md b/docs/contributing/README.md
new file mode 100644
index 000000000..4f7a7c06b
--- /dev/null
+++ b/docs/contributing/README.md
@@ -0,0 +1,57 @@
+# Malcolm Contributor Guide
+
+This document is a work in progress to cover the topics listed in [idaholab/Malcolm#25](https://github.com/idaholab/Malcolm/issues/25).
+
+The purpose of this document is to provide some direction for those willing to modify Malcolm, whether for local customization or for contribution to the Malcolm project.
+
+## Local modifications
+
+### Docker bind mounts
+
+### Building Malcolm's Docker images
+
+## Zeek
+
+### `local.zeek`
+
+### Adding a new Zeek package
+
+## Logstash
+
+### Parsing new Zeek logs
+
+### Parsing a new log data source
+
+### Enrichments
+
+### Dynamic pipelines
+
+### Logstash plugins
+
+## Kibana
+
+### Adding new visualizations and dashboards
+
+### Kibana plugins
+
+## Moloch
+
+### Adding new fields to WISE
+
+## PCAP processors
+
+## Carved file scanners
+
+## Adding a new service (Docker image)
+
+### Networking and firewall
+
+## Copyright
+
+[Malcolm](https://github.com/cisagov/Malcolm) is Copyright 2021 Battelle Energy Alliance, LLC, and is developed and released through the cooperation of the [Cybersecurity and Infrastructure Security Agency](https://www.cisa.gov/) of the [U.S. Department of Homeland Security](https://www.dhs.gov/).
+
+See [`License.txt`](../../License.txt) for the terms of its release.
+
+### Contact information of author(s):
+
+[malcolm@inl.gov](mailto:malcolm@inl.gov?subject=Malcolm)
diff --git a/file-upload/nginx/sites-available/default b/file-upload/nginx/sites-available/default
index 02170cda3..362a6cb1b 100644
--- a/file-upload/nginx/sites-available/default
+++ b/file-upload/nginx/sites-available/default
@@ -21,7 +21,7 @@ server {
fastcgi_send_timeout 300s;
fastcgi_busy_buffers_size 384k;
fastcgi_request_buffering off;
- fastcgi_pass unix:/run/php/php7.3-fpm.sock;
+ fastcgi_pass unix:/run/php/php7.4-fpm.sock;
}
location ~ /\.ht {
diff --git a/file-upload/supervisord.conf b/file-upload/supervisord.conf
index 6197c3a46..849658bc3 100644
--- a/file-upload/supervisord.conf
+++ b/file-upload/supervisord.conf
@@ -25,7 +25,7 @@ stdout_logfile_maxbytes=0
redirect_stderr=true
[program:php]
-command=php-fpm7.3 -F -R -g /tmp/php-fpm.pid
+command=php-fpm7.4 -F -R -g /tmp/php-fpm.pid
stdout_logfile=/dev/fd/1
stdout_logfile_maxbytes=0
redirect_stderr=true
diff --git a/htadmin/htadmin.sh b/htadmin/htadmin.sh
index 10da60f22..8ab93a399 100644
--- a/htadmin/htadmin.sh
+++ b/htadmin/htadmin.sh
@@ -17,6 +17,6 @@ else