Cherry-pick #24570 to 7.x: [Filebeat] Add Malware Bazaar to Threat Intel Module

filebeat/docs/fields.asciidoc

@@ -145017,7 +145017,7 @@ type: keyword
 --
 
 
-*`threatintel.indicator.geo.geo.city_name`*::
+*`threatintel.indicator.geo.city_name`*::
 +
 --
 City name.
@@ -145028,7 +145028,7 @@ example: Montreal
 
 --
 
-*`threatintel.indicator.geo.geo.country_iso_code`*::
+*`threatintel.indicator.geo.country_iso_code`*::
 +
 --
 Country ISO code.
@@ -145039,7 +145039,7 @@ example: CA
 
 --
 
-*`threatintel.indicator.geo.geo.country_name`*::
+*`threatintel.indicator.geo.country_name`*::
 +
 --
 Country name.
@@ -145050,7 +145050,7 @@ example: Canada
 
 --
 
-*`threatintel.indicator.geo.geo.location`*::
+*`threatintel.indicator.geo.location`*::
 +
 --
 Longitude and latitude.
@@ -145061,7 +145061,7 @@ example: { "lon": -73.614830, "lat": 45.505918 }
 
 --
 
-*`threatintel.indicator.geo.geo.region_iso_code`*::
+*`threatintel.indicator.geo.region_iso_code`*::
 +
 --
 Region ISO code.
@@ -145072,7 +145072,7 @@ example: CA-QC
 
 --
 
-*`threatintel.indicator.geo.geo.region_name`*::
+*`threatintel.indicator.geo.region_name`*::
 +
 --
 Region name.
@@ -145142,6 +145142,16 @@ type: keyword
 The file's sha256 hash, if available.
 
 
+type: keyword
+
+--
+
+*`threatintel.indicator.file.hash.sha384`*::
++
+--
+The file's sha384 hash, if available.
+
+
 type: keyword
 
 --
@@ -145159,7 +145169,7 @@ type: keyword
 *`threatintel.indicator.file.type`*::
 +
 --
-The file type
+The file type.
 
 
 type: keyword
@@ -145169,7 +145179,7 @@ type: keyword
 *`threatintel.indicator.file.size`*::
 +
 --
-The file's total size
+The file's total size.
 
 
 type: long
@@ -145179,7 +145189,27 @@ type: long
 *`threatintel.indicator.file.name`*::
 +
 --
-The file's name
+The file's name.
+
+
+type: keyword
+
+--
+
+*`threatintel.indicator.file.extension`*::
++
+--
+The file's extension.
+
+
+type: keyword
+
+--
+
+*`threatintel.indicator.file.mime_type`*::
++
+--
+The file's MIME type.
 
 
 type: keyword
@@ -145374,6 +145404,16 @@ example: *.elastic.co
 
 --
 
+*`threatintel.indicator.signature`*::
++
+--
+Malware family of sample (if available).
+
+
+type: keyword
+
+--
+
 [float]
 === abusemalware
 
@@ -145661,6 +145701,105 @@ type: keyword
 The STIX reference object.
 
 
+type: keyword
+
+--
+
+[float]
+=== malwarebazaar
+
+Fields for Malware Bazaar Threat Intel
+
+
+
+*`threatintel.malwarebazaar.file_type`*::
++
+--
+File type guessed by Malware Bazaar.
+
+
+type: keyword
+
+--
+
+*`threatintel.malwarebazaar.signature`*::
++
+--
+Malware familiy.
+
+
+type: keyword
+
+--
+
+*`threatintel.malwarebazaar.tags`*::
++
+--
+A list of tags associated with the queried malware sample.
+
+
+type: keyword
+
+--
+
+
+*`threatintel.malwarebazaar.intelligence.downloads`*::
++
+--
+Number of downloads from MalwareBazaar.
+
+
+type: long
+
+--
+
+*`threatintel.malwarebazaar.intelligence.uploads`*::
++
+--
+Number of uploads from MalwareBazaar.
+
+
+type: long
+
+--
+
+
+*`threatintel.malwarebazaar.intelligence.mail.Generic`*::
++
+--
+Malware seen in generic spam traffic.
+
+
+type: keyword
+
+--
+
+*`threatintel.malwarebazaar.intelligence.mail.IT`*::
++
+--
+Malware seen in IT spam traffic.
+
+
+type: keyword
+
+--
+
+*`threatintel.malwarebazaar.anonymous`*::
++
+--
+Identifies if the sample was submitted anonymously.
+
+
+type: long
+
+--
+
+*`threatintel.malwarebazaar.code_sign`*::
++
+--
+Code signing information for the sample.
+
+
 type: keyword
 
 --

filebeat/docs/modules/threatintel.asciidoc

@@ -20,11 +20,13 @@ Processors]. The related threat intel attribute that is meant to be used for
 matching incoming source data is stored under the `threatintel.indicator.*`
 fields.
 
-The available filesets are:
+[float]
+=== The available filesets are:
 
-* `abuseurl`: Supports URL entities from Abuse.ch.
-* `abusemalware`: Supports Malware/Payload entities from Abuse.ch.
+* `abuseurl`: Supports gathering URL entities from Abuse.ch.
+* `abusemalware`: Supports gathering Malware/Payload entities from Abuse.ch.
 * `misp`: Supports gathering threat intel attributes from MISP (replaces MISP module).
+* `malwarebazaar`: Supports gathering Malware/Payload entities from Malware Bazaar.
 * `otx`: Supports gathering threat intel attributes from AlientVault OTX.
 * `anomali`: Supports gathering threat intel attributes from Anomali.
 
@@ -108,6 +110,60 @@ Abuse.ch Malware Threat Intel is mapped to the following ECS fields.
 | file_size                  | threatintel.indicator.file.size
 |================================================================
 
+[float]
+==== `malwarebazaar` fileset settings
+
+This fileset contacts the Malware Bazaar API and fetches all new malicious hashes found
+the last 10 minutes.
+
+To configure the module, please utilize the default URL unless specified as the
+example below:
+
+[source,yaml]
+----
+- module: threatintel
+  malwarebazaar:
+    enabled: true
+    var.input: httpjson
+    var.url: https://mb-api.abuse.ch/api/v1/
+    var.interval: 10m
+----
+
+include::../include/var-paths.asciidoc[]
+
+*`var.url`*::
+
+The URL of the API endpoint to connect with.
+
+*`var.interval`*::
+
+How often the API is polled for updated information.
+
+Malware Bazaar Threat Intel is mapped to the following ECS fields.
+
+[options="header"]
+|================================================================
+| Malware Threat IntelFields | ECS Fields
+| md5_hash                   | threatintel.indicator.file.hash.md5
+| sha256_hash                | threatintel.indicator.file.hash.sha256
+| tlsh                       | threatintel.indicator.file.hash.tlsh
+| ssdeep                     | threatintel.indicator.file.hash.ssdeep
+| imphash                    | threatintel.indicator.file.pe.imphash
+| file_size                  | threatintel.indicator.file.size
+| file_name                  | threatintel.indicator.file.name
+| file_type_mime             | threatintel.indicator.file.mime_type
+| file_type                  | threatintel.indicator.file.type
+| reporter                   | threatintel.indicator.provider
+| origin_country             | threatintel.indicator.geo.country_iso_code
+| signature                  | threatintel.indicator.signature
+| code_sign.subject_cn       | threatintel.indicator.file.x509.subject.common_name
+| code_sign.issuer_cn        | threatintel.indicator.file.x509.issuer.common_name
+| code_sign.algorithm        | threatintel.indicator.file.x509.public_key_algorithm
+| code_sign.valid_from       | threatintel.indicator.file.x509.not_before
+| code_sign.valid_to         | threatintel.indicator.file.x509.not_after
+| code_sign.serial_number    | threatintel.indicator.file.x509.serial_number
+|================================================================
+
 [float]
 ==== `misp` fileset settings
 

filebeat/tests/system/test_modules.py

@@ -282,8 +282,10 @@ def clean_keys(obj):
         "threatintel.abuseurl",
         "threatintel.abusemalware",
         "threatintel.anomali",
+        "threatintel.malwarebazaar",
         "snyk.vulnerabilities",
-        "awsfargate.log"
+        "snyk.audit",
+        "awsfargate.log",
     }
     # dataset + log file pairs for which @timestamp is kept as an exception from above
     remove_timestamp_exception = {

x-pack/filebeat/filebeat.reference.yml

@@ -2151,6 +2151,18 @@ filebeat.modules:
     # The interval to poll the API for updates.
     var.interval: 10m
 
+  malwarebazaar:
+    enabled: true
+
+    # Input used for ingesting threat intel data.
+    var.input: httpjson
+
+    # The URL used for Threat Intel API calls.
+    var.url: https://mb-api.abuse.ch/api/v1/
+
+    # The interval to poll the API for updates.
+    var.interval: 10m
+
   misp:
     enabled: true
 
@@ -2170,7 +2182,7 @@ filebeat.modules:
     # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context.
     # For examples please reference the filebeat module documentation.
     #var.filters:
-    #  - threat_level: [4, 5] 
+    #  - threat_level: [4, 5]
     #  - to_ids: true
 
     # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer
@@ -2206,6 +2218,8 @@ filebeat.modules:
 
     # The interval to poll the API for updates
     var.interval: 5m
+
+=======
 
   anomali:
     enabled: true

x-pack/filebeat/module/snyk/vulnerabilities/test/snyk_vulns.ndjson.log-expected.json

@@ -1,5 +1,6 @@
 [
     {
+        "@timestamp": "2021-04-20T09:23:37.189Z",
         "event.dataset": "snyk.vulnerabilities",
         "event.module": "snyk",
         "event.timezone": "-02:00",
@@ -99,6 +100,7 @@
         "vulnerability.severity": "high"
     },
     {
+        "@timestamp": "2021-04-20T09:23:37.189Z",
         "event.dataset": "snyk.vulnerabilities",
         "event.module": "snyk",
         "event.timezone": "-02:00",
@@ -200,6 +202,7 @@
         "vulnerability.severity": "high"
     },
     {
+        "@timestamp": "2021-04-20T09:23:37.190Z",
         "event.dataset": "snyk.vulnerabilities",
         "event.module": "snyk",
         "event.timezone": "-02:00",
@@ -295,6 +298,7 @@
         "vulnerability.severity": "high"
     },
     {
+        "@timestamp": "2021-04-20T09:23:37.190Z",
         "event.dataset": "snyk.vulnerabilities",
         "event.module": "snyk",
         "event.timezone": "-02:00",