Malcolm v3.0.0 release development (#160)

* bring idaholab/Malcolm up-to-date with cisagov/Malcolm

* Restore malcolm.fyi web page generation

* ISO sha256sum placeholders

* working on idaholab#15, migrating to open distro for elastic

* bump yara to 4.0.5

* update SHA256 sums

* don't show comments from config file in settings appl7

* use mmguero-dev/zeek-community-id so it will build with docker 3.0.12

* somehow the extracted file http server stuff didn't make it into this branch

* expose HTTP file server port

* added filemon virtual_host

* added depends for nginx upstream

* set hostname to file-monitor for filemon

* Allow overriding cloned version of a plugin with a specific branch, and for corelight/bro-xor-exe-plugin use release 1.2 (pre commit corelight/zeek-xor-exe-plugin@a86f595) for Zeek 3.0.x and the latest release for Zeek 4.x

* use corelight/zeek-community-id build 3.0.0 for zeek 3.0.x

* put URLs in quotes for zeek clone

* temporarily remove the 'Top Actions and Results by Service' visualization (of type kbn_sankey) as it is not compatible with the current version of Kibana being used. tracking externally at uniberg/kbn_sankey_vis#15, at which point we can add it back in

* updates for open distro-based images for using non-1000 UIDs

* updates for open distro-based images for using non-1000 UIDs

* tweaks to iso for 3.0.0

* remove unnecessary package

* updated dashboards for new Kibana for Malcolm 3.0.0

* fix wording of prompt for email account inforrmation for alerts

* fix some dashboards where i lost a comma in the JSON

* updates to Arkime startup to try to fix race condition with fields getting created

* update auditbeat.yml rules to ignore firefox caching stuff (reduce noise)

* fix spacing in WISE field definition

* fixed a few dashboards (tag cloud visualizations didn't come over correctly)

* continuation of the idea for commit 045d7e6, for Logstash the STDIN doesn't work if we are dropping privs, which we don't need to do anyway. So differentiate between elastic and logstash for storing keystore values when dropping privileges

* minor tweaks to cloud visualizations

* updated local.zeek for sensor image to include ICSNPP parsers

* Revert "updated local.zeek for sensor image to include ICSNPP parsers"

This reverts commit dc19d83.

* updated local.zeek for sensor image to include ICSNPP parsers

* Updates to documentation and comments in preparation for 3.0.0 release

.gitignore

@@ -15,7 +15,6 @@
 /nginx/nginx_ldap.conf
 /htadmin/config.ini
 /htadmin/metadata
-/elastalert/config/smtp-auth.yaml
 
 # development
 .vagrant

Dockerfiles/arkime.Dockerfile

@@ -214,10 +214,8 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \
     ln -sfr /data/pcap_moloch_and_zeek_processor.py /data/pcap_moloch_processor.py && \
     cp -f /data/moloch_update_geo.sh $ARKIMEDIR/bin/moloch_update_geo.sh && \
     chmod u+s $ARKIMEDIR/bin/moloch-capture && \
-    chown -R $PUSER:$PGROUP $ARKIMEDIR/logs && \
     mkdir -p /var/run/moloch && \
-    chown -R $PUSER:$PGROUP /var/run/moloch
-
+    chown -R $PUSER:$PGROUP $ARKIMEDIR/etc $ARKIMEDIR/logs /var/run/moloch
 #Update Path
 ENV PATH="/data:$ARKIMEDIR/bin:${PATH}"
 

Dockerfiles/elasticsearch.Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/elasticsearch/elasticsearch-oss:7.6.2
+FROM amazon/opendistro-for-elasticsearch:1.12.0
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -7,8 +7,8 @@ LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
 LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
 LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
 LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
-LABEL org.opencontainers.image.title='malcolmnetsec/elasticsearch-oss'
-LABEL org.opencontainers.image.description='Malcolm container providing Elasticsearch (the Apache-licensed variant)'
+LABEL org.opencontainers.image.title='malcolmnetsec/elasticsearch-od'
+LABEL org.opencontainers.image.description='Malcolm container providing Elasticsearch (the Apache-licensed Open Distro variant)'
 
 ARG DEFAULT_UID=1000
 ARG DEFAULT_GID=1000
@@ -20,11 +20,23 @@ ENV PUSER_PRIV_DROP true
 
 ENV TERM xterm
 
+ARG GITHUB_OAUTH_TOKEN=""
+ARG DISABLE_INSTALL_DEMO_CONFIG=true
+ENV DISABLE_INSTALL_DEMO_CONFIG $DISABLE_INSTALL_DEMO_CONFIG
+
+# Malcolm manages authentication and encryption via NGINX reverse proxy
+# https://opendistro.github.io/for-elasticsearch-docs/docs/security/configuration/disable/
+# https://opendistro.github.io/for-elasticsearch-docs/docs/install/docker/#customize-the-docker-image
+# https://github.com/opendistro-for-elasticsearch/opendistro-build/issues/613
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security && \
+    /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_performance_analyzer && \
+  echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0' > /usr/share/elasticsearch/config/elasticsearch.yml && \
+  chown -R $PUSER:$PGROUP /usr/share/elasticsearch/config/elasticsearch.yml && \
+  sed -i "s/\b1000\b/\${PUID:-${DEFAULT_UID}}/g" /usr/local/bin/docker-entrypoint.sh
+
+# just used for initial keystore creation
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 
-ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/usr/local/bin/docker-entrypoint.sh"]
-
-
 # to be populated at build-time:
 ARG BUILD_DATE
 ARG MALCOLM_VERSION

Dockerfiles/file-monitor.Dockerfile

@@ -75,7 +75,7 @@ ENV EXTRACTED_FILE_ENABLE_CAPA $EXTRACTED_FILE_ENABLE_CAPA
 ENV EXTRACTED_FILE_CAPA_VERBOSE $EXTRACTED_FILE_CAPA_VERBOSE
 ENV SRC_BASE_DIR "/usr/local/src"
 ENV CLAMAV_RULES_DIR "/var/lib/clamav"
-ENV YARA_VERSION "4.0.4"
+ENV YARA_VERSION "4.0.5"
 ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
 ENV YARA_RULES_URL "https://github.com/Neo23x0/signature-base"
 ENV YARA_RULES_DIR "/yara-rules"
@@ -228,6 +228,7 @@ VOLUME ["$YARA_RULES_DIR"]
 VOLUME ["$YARA_RULES_SRC_DIR"]
 
 EXPOSE 3310
+EXPOSE $EXTRACTED_FILE_HTTP_SERVER_PORT
 
 ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh", "/docker-entrypoint.sh"]
 

Dockerfiles/filebeat.Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:7.6.2
+FROM docker.elastic.co/beats/filebeat-oss:7.10.0
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

Dockerfiles/kibana-helper.Dockerfile

@@ -0,0 +1,92 @@
+FROM alpine:3.12
+
+# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
+LABEL maintainer="[email protected]"
+LABEL org.opencontainers.image.authors='[email protected]'
+LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
+LABEL org.opencontainers.image.documentation='https://github.com/cisagov/Malcolm/blob/master/README.md'
+LABEL org.opencontainers.image.source='https://github.com/cisagov/Malcolm'
+LABEL org.opencontainers.image.vendor='Cybersecurity and Infrastructure Security Agency'
+LABEL org.opencontainers.image.title='malcolmnetsec/kibana-helper'
+LABEL org.opencontainers.image.description='Malcolm container providing Kibana support functions'
+
+ARG DEFAULT_UID=1000
+ARG DEFAULT_GID=1000
+ENV DEFAULT_UID $DEFAULT_UID
+ENV DEFAULT_GID $DEFAULT_GID
+ENV PUSER "helper"
+ENV PGROUP "helper"
+ENV PUSER_PRIV_DROP true
+
+ENV TERM xterm
+
+ARG ARKIME_INDEX_PATTERN="sessions2-*"
+ARG ARKIME_INDEX_PATTERN_ID="sessions2-*"
+ARG ARKIME_INDEX_TIME_FIELD="firstPacket"
+ARG CREATE_ES_ARKIME_SESSION_INDEX="true"
+ARG ELASTICSEARCH_URL="http://elasticsearch:9200"
+ARG ISM_SNAPSHOT_COMPRESSED=false
+ARG ISM_SNAPSHOT_REPO=logs
+ARG KIBANA_OFFLINE_REGION_MAPS_PORT="28991"
+ARG KIBANA_URL="http://kibana:5601/kibana"
+
+ENV ARKIME_INDEX_PATTERN $ARKIME_INDEX_PATTERN
+ENV ARKIME_INDEX_PATTERN_ID $ARKIME_INDEX_PATTERN_ID
+ENV ARKIME_INDEX_TIME_FIELD $ARKIME_INDEX_TIME_FIELD
+ENV CREATE_ES_ARKIME_SESSION_INDEX $CREATE_ES_ARKIME_SESSION_INDEX
+ENV ELASTICSEARCH_URL $ELASTICSEARCH_URL
+ENV ISM_SNAPSHOT_COMPRESSED $ISM_SNAPSHOT_COMPRESSED
+ENV ISM_SNAPSHOT_REPO $ISM_SNAPSHOT_REPO
+ENV KIBANA_OFFLINE_REGION_MAPS_PORT $KIBANA_OFFLINE_REGION_MAPS_PORT
+ENV KIBANA_URL $KIBANA_URL
+ENV PATH="/data:${PATH}"
+
+ENV SUPERCRONIC_VERSION "0.1.12"
+ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
+ENV SUPERCRONIC "supercronic-linux-amd64"
+ENV SUPERCRONIC_SHA1SUM "048b95b48b708983effb2e5c935a1ef8483d9e3e"
+ENV SUPERCRONIC_CRONTAB "/etc/crontab"
+
+ADD kibana/dashboards /opt/kibana/dashboards
+ADD kibana/maps /opt/maps
+ADD kibana/scripts /data/
+ADD kibana/supervisord.conf /etc/supervisord.conf
+ADD kibana/zeek_template.json /data/zeek_template.json
+ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
+ADD shared/bin/elastic_search_status.sh /data/
+ADD shared/bin/elastic_index_size_prune.py /data/
+
+RUN apk --no-cache add bash python3 py3-pip curl procps psmisc npm shadow jq && \
+    npm install -g http-server && \
+    pip3 install supervisor humanfriendly && \
+    curl -fsSLO "$SUPERCRONIC_URL" && \
+      echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
+      chmod +x "$SUPERCRONIC" && \
+      mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
+      ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
+    addgroup -g ${DEFAULT_GID} ${PGROUP} ; \
+      adduser -D -H -u ${DEFAULT_UID} -h /nonexistant -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \
+      addgroup ${PUSER} tty ; \
+      addgroup ${PUSER} shadow ; \
+    mkdir -p /data/init && \
+    chown -R ${PUSER}:${PGROUP} /opt/kibana/dashboards /opt/maps /data/init && \
+    chmod 755 /data/*.sh /data/*.py /data/init && \
+    chmod 400 /opt/maps/* && \
+    (echo -e "*/2 * * * * /data/kibana-create-moloch-sessions-index.sh\n0 10 * * * /data/kibana_index_refresh.py --template zeek_template\n*/20 * * * * /data/elastic_index_size_prune.py" > ${SUPERCRONIC_CRONTAB})
+
+EXPOSE $KIBANA_OFFLINE_REGION_MAPS_PORT
+
+ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
+
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf", "-n"]
+
+VOLUME ["/data/init"]
+
+# to be populated at build-time:
+ARG BUILD_DATE
+ARG MALCOLM_VERSION
+ARG VCS_REVISION
+
+LABEL org.opencontainers.image.created=$BUILD_DATE
+LABEL org.opencontainers.image.version=$MALCOLM_VERSION
+LABEL org.opencontainers.image.revision=$VCS_REVISION