Merge branch 'development' of https://github.com/mmguero-dev/Malcolm …

…into v642_merge_cisagov

Dockerfiles/dashboards.Dockerfile

@@ -14,10 +14,10 @@ ENV PGROUP "dashboarder"
 
 ENV TERM xterm
 
-ARG OPENSEARCH_VERSION="2.3.0"
+ARG OPENSEARCH_VERSION="2.4.0"
 ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION
 
-ARG OPENSEARCH_DASHBOARDS_VERSION="2.3.0"
+ARG OPENSEARCH_DASHBOARDS_VERSION="2.4.0"
 ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION
 
 # base system dependencies for checking out and building plugins
@@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \
 
 # runtime ##################################################################
 
-FROM opensearchproject/opensearch-dashboards:2.3.0
+FROM opensearchproject/opensearch-dashboards:2.4.0
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'

Dockerfiles/filebeat.Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:7.10.2
+FROM docker.elastic.co/beats/filebeat-oss:8.5.1
 
 # Copyright (c) 2022 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -21,6 +21,7 @@ ENV PGROUP "filebeat"
 # can chown uploaded files
 ENV PUSER_PRIV_DROP false
 
+ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
 ARG AUTO_TAG=true
@@ -68,18 +69,37 @@ USER root
 
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
 
-RUN yum install -y epel-release && \
-        yum upgrade -y && \
-        yum install -y curl inotify-tools file psmisc tar gzip unzip cpio bzip2 lzma xz openssl p7zip p7zip-plugins unar python3-setuptools python3-pip && \
-        yum clean all && \
-    ln -sr /usr/sbin/fuser /bin/fuser && \
+RUN apt-get -q update && \
+    apt-get -y -q --no-install-recommends upgrade && \
+    apt-get -y --no-install-recommends install \
+        bzip2 \
+        cpio \
+        curl \
+        file \
+        gzip \
+        inotify-tools \
+        lzma \
+        openssl \
+        p7zip \
+        p7zip-full \
+        p7zip-rar \
+        psmisc \
+        python3-pip \
+        python3-setuptools \
+        tar \
+        unar \
+        unzip \
+        xz-utils && \
     python3 -m pip install patool entrypoint2 pyunpack python-magic ordered-set supervisor && \
     curl -fsSLO "$SUPERCRONIC_URL" && \
       echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
       chmod +x "$SUPERCRONIC" && \
       mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
       ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
-    chmod +x /usr/bin/tini
+    chmod +x /usr/bin/tini && \
+    apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD filebeat/filebeat.yml /usr/share/filebeat/filebeat.yml
@@ -97,7 +117,7 @@ RUN for INPUT in nginx tcp; do \
       chmod 770 /usr/share/filebeat-$INPUT/data; \
     done; \
     chmod 755 /usr/local/bin/*.sh /usr/local/bin/*.py && \
-    (echo -e "* * * * * /usr/local/bin/filebeat-process-zeek-folder.sh\n*/5 * * * * /usr/local/bin/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB})
+    (echo "* * * * * /usr/local/bin/filebeat-process-zeek-folder.sh\n*/5 * * * * /usr/local/bin/filebeat-clean-zeeklogs-processed-folder.py" > ${SUPERCRONIC_CRONTAB})
 
 ENV AUTO_TAG $AUTO_TAG
 ENV LOG_CLEANUP_MINUTES $LOG_CLEANUP_MINUTES

Dockerfiles/logstash.Dockerfile

@@ -13,7 +13,7 @@ RUN amazon-linux-extras install -y epel && \
       glibc-devel \
       glibc-headers \
       java-1.8.0-openjdk \
-      java-latest-openjdk-devel \
+      java-1.8.0-openjdk-devel \
       libffi-devel \
       libtool \
       libyaml-devel \
@@ -44,7 +44,7 @@ RUN cd /opt && \
     sed -i "s/\('logstash-mixin-ecs_compatibility_support'\),.*/\1/" ./logstash-filter-fingerprint/logstash-filter-fingerprint.gemspec && \
     /bin/bash -lc "export LS_JAVA_HOME=$(realpath $(dirname $(find /usr/lib/jvm -name javac -type f))/../) && cd /opt/logstash-filter-fingerprint && ( bundle install || bundle install ) && gem build logstash-filter-fingerprint.gemspec && bundle info logstash-filter-fingerprint"
 
-FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.3
+FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.4.0
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
@@ -63,6 +63,7 @@ ENV PUSER "logstash"
 ENV PGROUP "logstash"
 ENV PUSER_PRIV_DROP true
 
+ENV DEBIAN_FRONTEND noninteractive
 ENV TERM xterm
 
 ENV TINI_VERSION v0.19.0
@@ -78,19 +79,24 @@ ENV LOGSTASH_PARSE_PIPELINE_ADDRESSES $LOGSTASH_PARSE_PIPELINE_ADDRESSES
 ENV LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_INTERNAL $LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_INTERNAL
 ENV LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_EXTERNAL $LOGSTASH_OPENSEARCH_PIPELINE_ADDRESS_EXTERNAL
 ENV LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES $LOGSTASH_OPENSEARCH_OUTPUT_PIPELINE_ADDRESSES
-ENV LS_JAVA_HOME=/usr/share/logstash/jdk
 
 USER root
 
 COPY --from=build /opt/logstash-filter-fingerprint /opt/logstash-filter-fingerprint
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
 
-RUN yum install -y epel-release && \
-    yum upgrade -y && \
-    yum install -y curl gettext patch python3-setuptools python3-pip python3-requests openssl && \
-    yum clean all && \
+RUN apt-get -q update && \
+    apt-get -y -q --no-install-recommends upgrade && \
+    apt-get -y --no-install-recommends install \
+        gettext \
+        patch \
+        python3-setuptools \
+        python3-pip \
+        python3-requests && \
     pip3 install ipaddress supervisor manuf pyyaml && \
-    /usr/share/logstash/bin/ruby -S gem install lru_redux && \
+    echo "gem 'lru_cache'" >> /usr/share/logstash/Gemfile && \
+    echo "gem 'rest-client'" >> /usr/share/logstash/Gemfile && \
+    /usr/share/logstash/bin/ruby -S bundle install && \
     logstash-plugin install --preserve logstash-filter-translate logstash-filter-cidr logstash-filter-dns \
                                        logstash-filter-json logstash-filter-prune logstash-filter-http \
                                        logstash-filter-grok logstash-filter-geoip logstash-filter-uuid \
@@ -99,7 +105,10 @@ RUN yum install -y epel-release && \
                                        logstash-input-beats logstash-output-elasticsearch && \
     logstash-plugin install /opt/logstash-filter-fingerprint/logstash-filter-fingerprint-*.gem && \
     chmod +x /usr/bin/tini && \
-    rm -rf /opt/logstash-filter-fingerprint /root/.cache /root/.gem /root/.bundle
+    rm -rf /opt/logstash-filter-fingerprint /root/.cache /root/.gem /root/.bundle && \
+    apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages autoremove && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/manuf-oui-parse.py /usr/local/bin/

Dockerfiles/opensearch.Dockerfile

@@ -1,4 +1,4 @@
-FROM opensearchproject/opensearch:2.3.0
+FROM opensearchproject/opensearch:2.4.0
 
 # Copyright (c) 2022 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

Dockerfiles/zeek.Dockerfile

@@ -30,8 +30,8 @@ ENV PGROUP "zeeker"
 ENV PUSER_PRIV_DROP false
 
 # for download and install
-ARG ZEEK_LTS=
-ARG ZEEK_VERSION=5.0.2-0
+ARG ZEEK_LTS=true
+ARG ZEEK_VERSION=5.0.3-0
 
 ENV ZEEK_LTS $ZEEK_LTS
 ENV ZEEK_VERSION $ZEEK_VERSION
@@ -165,8 +165,8 @@ ADD shared/bin/nic-capture-setup.sh /usr/local/bin/
 # these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh
 ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22
 ENV ZEEK_THIRD_PARTY_PLUGINS_GREP  "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY__OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|ANALYZER_SPICY_GENISYS_TCP|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)"
-ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 22
-ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP  "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
+ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 23
+ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP  "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
 
 RUN mkdir -p /tmp/logs && \
     cd /tmp/logs && \

Gemfile

@@ -2,4 +2,5 @@ source "https://rubygems.org"
 
 gem "github-pages", group: :jekyll_plugins
 gem "jekyll-include-cache", group: :jekyll_plugins
-gem 'jekyll-seo-tag', group: :jekyll_plugins
+gem 'jekyll-seo-tag', group: :jekyll_plugins
+gem 'jekyll-sitemap', group: :jekyll_plugins

README.md

@@ -32,5 +32,3 @@ See `License.txt` for the terms of its release.
 ## <a name="Contact"></a>Contact information of author(s):
 
 [[email protected]](mailto:[email protected]?subject=Malcolm)
-
-[![Join the chat at https://gitter.im/malcolmnetsec/community](https://badges.gitter.im/malcolmnetsec/community.svg)](https://gitter.im/malcolmnetsec/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)

_config.yml

@@ -5,6 +5,9 @@ logo: docs/images/logo/Malcolm_outline_banner_dark.png
 remote_theme: pages-themes/[email protected]
 external_download_url: https://malcolm.fyi/docs/download.html
 youtube_url: https://www.youtube.com/c/MalcolmNetworkTrafficAnalysisToolSuite
+mastodon:
+  id:
+  url:
 docs_uri: docs/
 alerting_docs_uri: docs/alerting.html
 anomaly_detection_docs_uri: docs/anomaly-detection.html
@@ -23,13 +26,15 @@ quickstart_docs_uri: docs/quickstart.html
 severity_docs_uri: docs/severity.html
 thirdparty_logs_docs_uri: docs/third-party-logs.html
 upload_docs_uri: docs/upload.html
+url:
 github:
   owner_name: Cybersecurity and Infrastructure Security Agency
   default_branch: main
 plugins:
   - jekyll-relative-links
   - jekyll-remote-theme
   - jekyll-seo-tag
+  - jekyll-sitemap
 show_downloads: true
 relative_links:
   enabled: true

_layouts/default.html

@@ -94,6 +94,10 @@ <h1><a href="{{ "/" | absolute_url }}">{{ site.title | default: site.github.repo
         <p class="view"><i class="fa fa-github" aria-hidden="true"></i> <a href="{{ site.github.owner_url }}">{{ site.github.owner_name }} on GitHub <i class="fa fa-sign-out" aria-hidden="true"></i></a></p>
         {% endif %}
         -->
+
+        {% if site.mastodon.id %}
+        <p class="view"><i class="fa fa-share-alt" aria-hidden="true"></i> <a rel="me" href="{{ site.mastodon.url }}">@{{ site.mastodon.id }} <i class="fa fa-sign-out" aria-hidden="true"></i></a></p>
+        {% endif %}
       </footer>
     </div>
     <script src="{{ "/assets/js/scale.fix.js" | relative_url }}"></script>

api/project/__init__.py

@@ -461,15 +461,15 @@ def bucketfield(fieldname, current_request, urls=None):
     response = s.execute()
     if (urls is not None) and (len(urls) > 0):
         return jsonify(
-            values=response.aggregations.to_dict()["values"],
+            values=response.aggregations.to_dict().get("values", {}),
             range=(start_time_ms // 1000, end_time_ms // 1000),
             filter=filters,
             fields=get_iterable(fieldname),
             urls=urls,
         )
     else:
         return jsonify(
-            values=response.aggregations.to_dict()["values"],
+            values=response.aggregations.to_dict().get("values", {}),
             range=(start_time_ms // 1000, end_time_ms // 1000),
             filter=filters,
             fields=get_iterable(fieldname),
@@ -530,7 +530,7 @@ def document(index):
     start_time_ms, end_time_ms, s = filtertime(s, args, default_from="1970-1-1", default_to="now")
     filters, s = filtervalues(s, args)
     return jsonify(
-        results=s.execute().to_dict()['hits']['hits'],
+        results=s.execute().to_dict().get('hits', {}).get('hits', []),
         range=(start_time_ms // 1000, end_time_ms // 1000),
         filter=filters,
     )
@@ -588,7 +588,7 @@ def fields():
             s = opensearch_dsl.Search(
                 using=opensearch_dsl.connections.get_connection(), index=app.config["ARKIME_FIELDS_INDEX"]
             ).extra(size=5000)
-            for hit in [x['_source'] for x in s.execute().to_dict()['hits']['hits']]:
+            for hit in [x['_source'] for x in s.execute().to_dict().get('hits', {}).get('hits', [])]:
                 if (fieldname := deep_get(hit, ['dbField2'])) and (fieldname not in fields):
                     if debugApi:
                         hit['source'] = 'arkime'

dashboards/opensearch_dashboards.yml

@@ -11,6 +11,7 @@ opensearch.username: "_MALCOLM_DASHBOARDS_OPENSEARCH_USER_"
 opensearch.password: "_MALCOLM_DASHBOARDS_OPENSEARCH_PASSWORD_"
 opensearch.ssl.verificationMode: "_MALCOLM_DASHBOARDS_OPENSEARCH_SSL_VERIFICATION_MODE_"
 opensearch.requestTimeout: 180000
+data_source.enabled: false
 
 opensearchDashboards.branding:
   applicationTitle: "Malcolm Dashboards"