From d970abaa8061144aa32a1ffb4be20a72819a31a1 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Fri, 8 Nov 2024 11:42:26 -0700 Subject: [PATCH] work in progress for mandiant threat intel integration, cisagov/Malcolm#358 --- config/zeek.env.example | 4 ++-- docs/capabilities-and-limitations.md | 2 +- docs/malcolm-config.md | 2 +- docs/zeek-intel.md | 2 +- .../config/hooks/normal/0169-pip-installs.hook.chroot | 1 + kubernetes/10-zeek.yml | 2 +- kubernetes/21-zeek-live.yml | 2 +- scripts/control.py | 2 +- shared/bin/zeek_intel_setup.sh | 2 +- 9 files changed, 10 insertions(+), 9 deletions(-) diff --git a/config/zeek.env.example b/config/zeek.env.example index 1ca4e171b..5c69531bf 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -7,11 +7,11 @@ ZEEK_LOCAL_NETS= ZEEK_JSON= # Specifies the value for Zeek's Intel::item_expiration timeout (-1min to disable) ZEEK_INTEL_ITEM_EXPIRATION=-1min -# When querying a TAXII or MISP feed, only process threat indicators that have +# When querying a threat intelligence feed, only process threat indicators that have # been created or modified since the time represented by this value; # it may be either a fixed date/time (01/01/2021) or relative interval (30 days ago) ZEEK_INTEL_FEED_SINCE= -# Whether or not to require SSL certificate verification when querying a TAXII or MISP feed +# Whether or not to require SSL certificate verification when querying an intelligence feed ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false # Number of threads to use for querying feeds for generating Zeek Intelligence Framework files ZEEK_INTEL_REFRESH_THREADS=2 diff --git a/docs/capabilities-and-limitations.md b/docs/capabilities-and-limitations.md index 7b13a9c29..9edb33f35 100644 --- a/docs/capabilities-and-limitations.md +++ b/docs/capabilities-and-limitations.md @@ -45,7 +45,7 @@ In short, Malcolm provides an easily deployable traffic analysis tool suite for - Limitation: Anomaly detection and machine learning algorithms rely on enough data (for network data, this generally means at least several weeks' worth or more) to be able to build a baseline of what is normal before they can accurately flag anomalies, and each network is different. Anomaly detection and ML are typically not useful for limited deployments without the available traffic to build that baseline. - Limitation: While Malcolm provides some powerful tools in the anomaly detection and ML realm, as of yet they have not been built out to provide the value that they will probably one day realize. * Threat ingestion - - Malcolm can ingest threat indicators in the form of static MISP- or STIX-formatted files. It can also subscribe to and periodically update threat indicators from [MISP](zeek-intel.md#ZeekIntelMISP) and [TAXII](zeek-intel.md#ZeekIntelSTIX) feeds. These indicators are converted into a format that is read by Zeek, and matches in network traffic are [surfaced through the Zeek intelligence framework](zeek-intel.md#ZeekIntel) for logging. + - Malcolm can ingest threat indicators in the form of static MISP- or STIX-formatted files. It can also subscribe to and periodically update threat indicators from [MISP](zeek-intel.md#ZeekIntelMISP), [TAXII](zeek-intel.md#ZeekIntelSTIX), and [Mandiant](zeek-intel.md#ZeekIntelMandiant) feeds. These indicators are converted into a format that is read by Zeek, and matches in network traffic are [surfaced through the Zeek intelligence framework](zeek-intel.md#ZeekIntel) for logging. - Limitation: Some formats for threat indicators allow for complex definitions and logic. For STIX/TAXII, only indicators of cyber-observable objects matched with the equals (=) comparison operator against a single value can be expressed as Zeek intelligence items. Similarly, only a subset of MISP attribute types can be expressed with the Zeek intelligence indicator types. While this is generally sufficient to cover most indicators interest, more complex indicators are silently ignored. * Network Modeling - Malcolm provides an instance of [NetBox](https://netboxlabs.com/oss/netbox/), an open-source "solution for modeling and documenting modern networks" which is used to model instrumented networks and enrich passively-observed network traffic from that model, a technique Malcolm calls ["Asset Interaction Analysis"](asset-interaction-analysis.md#AssetInteractionAnalysis). Users can pivot between the network visualization tools (the Asset Interaction Analysis and Zeek Known Summary dashboards in OpenSearch Dashboards, and the Arkime Sessions interface) and the NetBox UI to investigate and examine network assets. diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index b598dc06e..55e2b7b96 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -127,7 +127,7 @@ Although the configuration script automates many of the following configuration - `ZEEK_DISABLE_ICS_ALL` and `ZEEK_DISABLE_ICS_…` - if set to `true`, these variables can be used to disable Zeek's protocol analyzers for Operational Technology/Industrial Control Systems (OT/ICS) protocols - `ZEEK_DISABLE_BEST_GUESS_ICS` - see ["Best Guess" Fingerprinting for ICS Protocols](ics-best-guess.md#ICSBestGuess) - `ZEEK_EXTRACTOR_MODE` – determines the file extraction behavior for file transfers detected by Zeek; see [Automatic file extraction and scanning](file-scanning.md#ZeekFileExtraction) for more details - - `ZEEK_INTEL_FEED_SINCE` - when querying a [TAXII](zeek-intel.md#ZeekIntelSTIX) or [MISP](zeek-intel.md#ZeekIntelMISP) feed, only process threat indicators created or modified since the time represented by this value; it may be either a fixed date/time (`01/01/2021`) or relative interval (`30 days ago`) + - `ZEEK_INTEL_FEED_SINCE` - when querying a [TAXII](zeek-intel.md#ZeekIntelSTIX), [MISP](zeek-intel.md#ZeekIntelMISP), or [Mandiant](zeek-intel.md#ZeekIntelMandiant) threat intelligence feed, only process threat indicators created or modified since the time represented by this value; it may be either a fixed date/time (`01/01/2021`) or relative interval (`30 days ago`) - `ZEEK_INTEL_ITEM_EXPIRATION` - specifies the value for Zeek's [`Intel::item_expiration`](https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#id-Intel::item_expiration) timeout as used by the [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel) (default `-1min`, which disables item expiration) - `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` - specifies a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) indicating the refresh interval for generating the [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel) files (defaults to empty, which disables automatic refresh) - `ZEEK_JA4SSH_PACKET_COUNT` - the Zeek [JA4+ plugin](https://github.com/FoxIO-LLC/ja4) calculates the JA4SSH value once for every *x* SSH packets; *x* is set here (default `200`) diff --git a/docs/zeek-intel.md b/docs/zeek-intel.md index 468f9529c..bdcc16b67 100644 --- a/docs/zeek-intel.md +++ b/docs/zeek-intel.md @@ -10,7 +10,7 @@ To quote Zeek's [Intelligence Framework](https://docs.zeek.org/en/master/framewo Malcolm doesn't come bundled with intelligence files from any particular feed, but they can be easily included into a local instance. On [startup]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_setup.sh), Malcolm's `ghcr.io/idaholab/malcolm/zeek` container enumerates the subdirectories under `./zeek/intel` (which is [bind mounted](https://docs.docker.com/storage/bind-mounts/) into the container's runtime) and configures Zeek so those intelligence files will be automatically included in its local policy. Subdirectories under `./zeek/intel` that contain their own `__load__.zeek` file will be `@load`-ed as-is, while subdirectories containing "loose" intelligence files will be [loaded](https://docs.zeek.org/en/master/frameworks/intel.html#loading-intelligence) automatically with a `redef Intel::read_files` directive. -Note that Malcolm does not manage updates for these intelligence files. Users use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX) or [MISP](#ZeekIntelMISP) feed as described below. +Note that Malcolm does not manage updates for these intelligence files. Users use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX), [MISP](#ZeekIntelMISP), or [Mandiant](#ZeekIntelMandiant) feed as described below. Adding and deleting intelligence files under this directory will take effect upon [restarting Malcolm](running.md#StopAndRestart). Alternately, users can use the `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` environment variable containing a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) to specify the interval at which the intel files should be refreshed. This can also be done manually without restarting Malcolm by running the following command from the Malcolm installation directory: diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 4fdaec171..f5a69daf0 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -13,6 +13,7 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta dateparser \ debinterface \ dominate \ + git+https://github.com/google/mandiant-ti-client \ humanfriendly \ pymisp \ python-dotenv \ diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index c08a95774..00b366dd0 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -71,7 +71,7 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" + value: "/data/config:zeek/intel/Mandiant,zeek/intel/MISP,zeek/intel/STIX;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" volumeMounts: - name: zeek-offline-intel-volume mountPath: "/data/config" diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 1997710fe..3045705ae 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -70,7 +70,7 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" + value: "/data/config:zeek/intel/Mandiant,zeek/intel/MISP,zeek/intel/STIX;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" volumeMounts: - name: zeek-live-intel-volume mountPath: "/data/config" diff --git a/scripts/control.py b/scripts/control.py index 204d10d60..2430685c8 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -1072,7 +1072,7 @@ def start(): BoundPath("zeek", "/zeek/extract_files", False, None, None), BoundPath("zeek", "/zeek/upload", False, None, None), BoundPath("zeek", "/opt/zeek/share/zeek/site/custom", False, None, None), - BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["MISP", "STIX"], None), + BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["Mandiant", "MISP", "STIX"], None), BoundPath("zeek-live", "/zeek/live", False, ["spool"], None), BoundPath( "filebeat", "/zeek", False, ["processed", "current", "live", "extract_files", "upload"], None diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index ca9dc1bb2..1d81ca9cc 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -95,7 +95,7 @@ EOF fi done - # process STIX and MISP inputs by converting them to Zeek intel format + # process STIX/MISP/Mandiant inputs by converting them to Zeek intel format if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] || [[ -r ./Mandiant/mandiant.yaml ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then "${THREAT_FEED_TO_ZEEK_SCRIPT}" \ --ssl-verify ${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION} \