Cherry-pick #24570 to 7.x: [Filebeat] Add Malware Bazaar to Threat In…

…tel Module (#25177) * [Filebeat] Add Malware Bazaar to Threat Intel Module (#24570) * fixed * update * Set content-type to form encoded * update config * dashboard and config work * test data * updated docs * dashboard screenshot * image location * ran mage fmt update * updated changelog * mage fmt * ran the tests * mage fmt after testing * added timestamp fix * fixed related.hash and tlsh * added elf.telfhash * mage'd everything * updated dashboard * Mage update * update snyk build to ignore timestamps * fixing test_modules.py timestamp * Add missing comma to array list item Co-authored-by: Derek Ditch <[email protected]> Co-authored-by: Marius Iversen <[email protected]> Co-authored-by: Adrian Serrano <[email protected]> Co-authored-by: Andrew Kroh <[email protected]> (cherry picked from commit 6a0dc39) * update reference file * mage fmt update * update docs typo Co-authored-by: Andrew Pease <[email protected]>
elastic · Apr 20, 2021 · 9e04a75 · 9e04a75
1 parent 22853a1
commit 9e04a75
Show file tree

Hide file tree

Showing 18 changed files with 3,638 additions and 30 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -145017,7 +145017,7 @@ type: keyword
 --
 
 
-*`threatintel.indicator.geo.geo.city_name`*::
+*`threatintel.indicator.geo.city_name`*::
 +
 --
 City name.
@@ -145028,7 +145028,7 @@ example: Montreal
 
 --
 
-*`threatintel.indicator.geo.geo.country_iso_code`*::
+*`threatintel.indicator.geo.country_iso_code`*::
 +
 --
 Country ISO code.
@@ -145039,7 +145039,7 @@ example: CA
 
 --
 
-*`threatintel.indicator.geo.geo.country_name`*::
+*`threatintel.indicator.geo.country_name`*::
 +
 --
 Country name.
@@ -145050,7 +145050,7 @@ example: Canada
 
 --
 
-*`threatintel.indicator.geo.geo.location`*::
+*`threatintel.indicator.geo.location`*::
 +
 --
 Longitude and latitude.
@@ -145061,7 +145061,7 @@ example: { "lon": -73.614830, "lat": 45.505918 }
 
 --
 
-*`threatintel.indicator.geo.geo.region_iso_code`*::
+*`threatintel.indicator.geo.region_iso_code`*::
 +
 --
 Region ISO code.
@@ -145072,7 +145072,7 @@ example: CA-QC
 
 --
 
-*`threatintel.indicator.geo.geo.region_name`*::
+*`threatintel.indicator.geo.region_name`*::
 +
 --
 Region name.
@@ -145142,6 +145142,16 @@ type: keyword
 The file's sha256 hash, if available.
 
 
+type: keyword
+
+--
+
+*`threatintel.indicator.file.hash.sha384`*::
++
+--
+The file's sha384 hash, if available.
+
+
 type: keyword
 
 --
@@ -145159,7 +145169,7 @@ type: keyword
 *`threatintel.indicator.file.type`*::
 +
 --
-The file type
+The file type.
 
 
 type: keyword
@@ -145169,7 +145179,7 @@ type: keyword
 *`threatintel.indicator.file.size`*::
 +
 --
-The file's total size
+The file's total size.
 
 
 type: long
@@ -145179,7 +145189,27 @@ type: long
 *`threatintel.indicator.file.name`*::
 +
 --
-The file's name
+The file's name.
+
+
+type: keyword
+
+--
+
+*`threatintel.indicator.file.extension`*::
++
+--
+The file's extension.
+
+
+type: keyword
+
+--
+
+*`threatintel.indicator.file.mime_type`*::
++
+--
+The file's MIME type.
 
 
 type: keyword
@@ -145374,6 +145404,16 @@ example: *.elastic.co
 
 --
 
+*`threatintel.indicator.signature`*::
++
+--
+Malware family of sample (if available).
+
+
+type: keyword
+
+--
+
 [float]
 === abusemalware
 
@@ -145661,6 +145701,105 @@ type: keyword
 The STIX reference object.
 
 
+type: keyword
+
+--
+
+[float]
+=== malwarebazaar
+
+Fields for Malware Bazaar Threat Intel
+
+
+
+*`threatintel.malwarebazaar.file_type`*::
++
+--
+File type guessed by Malware Bazaar.
+
+
+type: keyword
+
+--
+
+*`threatintel.malwarebazaar.signature`*::
++
+--
+Malware familiy.
+
+
+type: keyword
+
+--
+
+*`threatintel.malwarebazaar.tags`*::
++
+--
+A list of tags associated with the queried malware sample.
+
+
+type: keyword
+
+--
+
+
+*`threatintel.malwarebazaar.intelligence.downloads`*::
++
+--
+Number of downloads from MalwareBazaar.
+
+
+type: long
+
+--
+
+*`threatintel.malwarebazaar.intelligence.uploads`*::
++
+--
+Number of uploads from MalwareBazaar.
+
+
+type: long
+
+--
+
+
+*`threatintel.malwarebazaar.intelligence.mail.Generic`*::
++
+--
+Malware seen in generic spam traffic.
+
+
+type: keyword
+
+--
+
+*`threatintel.malwarebazaar.intelligence.mail.IT`*::
++
+--
+Malware seen in IT spam traffic.
+
+
+type: keyword
+
+--
+
+*`threatintel.malwarebazaar.anonymous`*::
++
+--
+Identifies if the sample was submitted anonymously.
+
+
+type: long
+
+--
+
+*`threatintel.malwarebazaar.code_sign`*::
++
+--
+Code signing information for the sample.
+
+
 type: keyword
 
 --

diff --git a/filebeat/docs/images/filebeat-threatintel-malware-bazaar.png b/filebeat/docs/images/filebeat-threatintel-malware-bazaar.png
diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc
@@ -20,11 +20,13 @@ Processors]. The related threat intel attribute that is meant to be used for
 matching incoming source data is stored under the `threatintel.indicator.*`
 fields.
 
-The available filesets are:
+[float]
+=== The available filesets are:
 
-* `abuseurl`: Supports URL entities from Abuse.ch.
-* `abusemalware`: Supports Malware/Payload entities from Abuse.ch.
+* `abuseurl`: Supports gathering URL entities from Abuse.ch.
+* `abusemalware`: Supports gathering Malware/Payload entities from Abuse.ch.
 * `misp`: Supports gathering threat intel attributes from MISP (replaces MISP module).
+* `malwarebazaar`: Supports gathering Malware/Payload entities from Malware Bazaar.
 * `otx`: Supports gathering threat intel attributes from AlientVault OTX.
 * `anomali`: Supports gathering threat intel attributes from Anomali.
 
@@ -108,6 +110,60 @@ Abuse.ch Malware Threat Intel is mapped to the following ECS fields.
 | file_size                  | threatintel.indicator.file.size
 |================================================================
 
+[float]
+==== `malwarebazaar` fileset settings
+
+This fileset contacts the Malware Bazaar API and fetches all new malicious hashes found
+the last 10 minutes.
+
+To configure the module, please utilize the default URL unless specified as the
+example below:
+
+[source,yaml]
+----
+- module: threatintel
+  malwarebazaar:
+    enabled: true
+    var.input: httpjson
+    var.url: https://mb-api.abuse.ch/api/v1/
+    var.interval: 10m
+----
+
+include::../include/var-paths.asciidoc[]
+
+*`var.url`*::
+
+The URL of the API endpoint to connect with.
+
+*`var.interval`*::
+
+How often the API is polled for updated information.
+
+Malware Bazaar Threat Intel is mapped to the following ECS fields.
+
+[options="header"]
+|================================================================
+| Malware Threat IntelFields | ECS Fields
+| md5_hash                   | threatintel.indicator.file.hash.md5
+| sha256_hash                | threatintel.indicator.file.hash.sha256
+| tlsh                       | threatintel.indicator.file.hash.tlsh
+| ssdeep                     | threatintel.indicator.file.hash.ssdeep
+| imphash                    | threatintel.indicator.file.pe.imphash
+| file_size                  | threatintel.indicator.file.size
+| file_name                  | threatintel.indicator.file.name
+| file_type_mime             | threatintel.indicator.file.mime_type
+| file_type                  | threatintel.indicator.file.type
+| reporter                   | threatintel.indicator.provider
+| origin_country             | threatintel.indicator.geo.country_iso_code
+| signature                  | threatintel.indicator.signature
+| code_sign.subject_cn       | threatintel.indicator.file.x509.subject.common_name
+| code_sign.issuer_cn        | threatintel.indicator.file.x509.issuer.common_name
+| code_sign.algorithm        | threatintel.indicator.file.x509.public_key_algorithm
+| code_sign.valid_from       | threatintel.indicator.file.x509.not_before
+| code_sign.valid_to         | threatintel.indicator.file.x509.not_after
+| code_sign.serial_number    | threatintel.indicator.file.x509.serial_number
+|================================================================
+
 [float]
 ==== `misp` fileset settings
 

diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -282,8 +282,10 @@ def clean_keys(obj):
         "threatintel.abuseurl",
         "threatintel.abusemalware",
         "threatintel.anomali",
+        "threatintel.malwarebazaar",
         "snyk.vulnerabilities",
-        "awsfargate.log"
+        "snyk.audit",
+        "awsfargate.log",
     }
     # dataset + log file pairs for which @timestamp is kept as an exception from above
     remove_timestamp_exception = {

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -2151,6 +2151,18 @@ filebeat.modules:
     # The interval to poll the API for updates.
     var.interval: 10m
 
+  malwarebazaar:
+    enabled: true
+
+    # Input used for ingesting threat intel data.
+    var.input: httpjson
+
+    # The URL used for Threat Intel API calls.
+    var.url: https://mb-api.abuse.ch/api/v1/
+
+    # The interval to poll the API for updates.
+    var.interval: 10m
+
   misp:
     enabled: true
 
@@ -2170,7 +2182,7 @@ filebeat.modules:
     # Optional filters that can be applied to the API for filtering out results. This should support the majority of fields in a MISP context.
     # For examples please reference the filebeat module documentation.
     #var.filters:
-    #  - threat_level: [4, 5] 
+    #  - threat_level: [4, 5]
     #  - to_ids: true
 
     # How far back to look once the beat starts up for the first time, the value has to be in hours. Each request afterwards will filter on any event newer
@@ -2206,6 +2218,8 @@ filebeat.modules:
 
     # The interval to poll the API for updates
     var.interval: 5m
+
+=======
 
   anomali:
     enabled: true

diff --git a/x-pack/filebeat/module/snyk/vulnerabilities/test/snyk_vulns.ndjson.log-expected.json b/x-pack/filebeat/module/snyk/vulnerabilities/test/snyk_vulns.ndjson.log-expected.json
@@ -1,5 +1,6 @@
 [
     {
+        "@timestamp": "2021-04-20T09:23:37.189Z",
         "event.dataset": "snyk.vulnerabilities",
         "event.module": "snyk",
         "event.timezone": "-02:00",
@@ -99,6 +100,7 @@
         "vulnerability.severity": "high"
     },
     {
+        "@timestamp": "2021-04-20T09:23:37.189Z",
         "event.dataset": "snyk.vulnerabilities",
         "event.module": "snyk",
         "event.timezone": "-02:00",
@@ -200,6 +202,7 @@
         "vulnerability.severity": "high"
     },
     {
+        "@timestamp": "2021-04-20T09:23:37.190Z",
         "event.dataset": "snyk.vulnerabilities",
         "event.module": "snyk",
         "event.timezone": "-02:00",
@@ -295,6 +298,7 @@
         "vulnerability.severity": "high"
     },
     {
+        "@timestamp": "2021-04-20T09:23:37.190Z",
         "event.dataset": "snyk.vulnerabilities",
         "event.module": "snyk",
         "event.timezone": "-02:00",