From f2e14f895974b55c6b39a133be6cfc4060848f64 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 15 Apr 2024 19:12:39 +1000 Subject: [PATCH] [filebeat][threatintel] MISP splitting fix for empty responses (#38917) Two fixes related to empty server responses: - Set `response.split.ignore_empty_value` to `true` to avoid indexing `{response:[]}`, which is the MISP server's empty response body. According to the `response.split` [documentation][1], "If the split target is empty the parent document will be kept. If documents with empty splits should be dropped, the `ignore_empty_value` option should be set to `true`." - Use the null-safe operator for a chained method invocation following a null-safe field access. All other null-safe operator usages were also reviewed and corrected where necessary. [1]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-split --------- Co-authored-by: Dan Kortschak (cherry picked from commit 692658c9fed090600bfdc067b0a44eff8f6dbe4c) --- CHANGELOG.next.asciidoc | 15 ++++++++ .../module/threatintel/misp/config/config.yml | 1 + .../threatintel/misp/ingest/pipeline.yml | 36 +++++++++---------- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 08806f8a68a..38b4ac283ad 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -49,6 +49,21 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094] - Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116] +- Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125] +- Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181] +- Fix the incorrect values generated by the uri_parts processor. {pull}38216[38216] +- Fix HTTPJSON handling of empty object bodies in POST requests. {issue}33961[33961] {pull}38290[38290] +- Fix PEM key validation for CEL and HTTPJSON inputs. {pull}38405[38405] +- Fix filebeat gcs input panic {pull}38407[38407] +- Rename `activity_guid` to `activity_id` in ETW input events to suit other Windows inputs. {pull}38530[38530] +- Add missing provider registration and fix published entity for Active Directory entityanalytics provider. {pull}38645[38645] +- Fix handling of un-parsed JSON in O365 module. {issue}37800[37800] {pull}38709[38709] +- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488] +- Fix indexing failures by re-enabling event normalisation in netflow input. {issue}38703[38703] {pull}38780[38780] +- Fix handling of truncated files in Filestream {issue}38070[38070] {pull}38416[38416] +- Fix panic when more than 32767 pipeline clients are active. {issue}38197[38197] {pull}38556[38556] +- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488] +- [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917] *Heartbeat* diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml index 9ad66efcf54..2f6a2f0e12f 100644 --- a/x-pack/filebeat/module/threatintel/misp/config/config.yml +++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml @@ -49,6 +49,7 @@ request.transforms: response.split: target: body.response + ignore_empty_value: true split: target: body.Event.Attribute ignore_empty_value: true diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml index 3cdc64d8703..8caba8e738c 100644 --- a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml @@ -63,12 +63,12 @@ processors: field: - misp.Attribute ignore_missing: true - if: ctx.misp?.Attribute.size() == 0 + if: ctx.misp?.Attribute?.size() == 0 - remove: field: - misp.Object ignore_missing: true - if: ctx.misp?.Object.size() == 0 + if: ctx.misp?.Object?.size() == 0 - date: field: misp.timestamp formats: @@ -144,12 +144,12 @@ processors: - set: field: threat.indicator.type value: file - if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type.startsWith('filename'))" + if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type?.startsWith('filename') == true" - rename: field: misp.attribute.value target_field: "threat.indicator.file.hash.{{misp.attribute.type}}" ignore_missing: true - if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp?.attribute?.type.startsWith('filename')" + if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp.attribute.type.startsWith('filename')" - rename: field: misp.attribute.value target_field: threat.indicator.file.name @@ -160,23 +160,23 @@ processors: patterns: - "%{WORD}\\|%{WORD:_tmp.hashtype}" ignore_missing: true - if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') + if: ctx.misp?.attribute?.type?.startsWith('filename|') == true - grok: field: misp.attribute.value patterns: - "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}" ignore_missing: true - if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') + if: ctx.misp?.attribute?.type?.startsWith('filename|') == true - set: field: threat.indicator.file.hash.{{_tmp.hashtype}} value: "{{_tmp.hashvalue}}" - if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null" + if: "ctx.misp?.attribute?.type?.startsWith('filename|') == true && ctx._tmp?.hashvalue != null && ctx._tmp?.hashtype != null" ## URL/URI indicator operations - set: field: threat.indicator.type value: url - if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)" + if: "['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)" - uri_parts: field: misp.attribute.value target_field: threat.indicator.url @@ -193,7 +193,7 @@ processors: - set: field: threat.indicator.type value: windows-registry-key - if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('regkey')" + if: "ctx.misp?.attribute?.type?.startsWith('regkey') == true" - rename: field: misp.attribute.value target_field: threat.indicator.registry.key @@ -210,7 +210,7 @@ processors: - set: field: threat.indicator.type value: autonomous-system - if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'" + if: "ctx.misp?.attribute?.type == 'AS'" - convert: field: misp.attribute.value type: long @@ -222,11 +222,11 @@ processors: - set: field: threat.indicator.type value: domain-name - if: "ctx.misp?.attribute?.type != null && (ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type.startsWith('domain'))" + if: "ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type?.startsWith('domain') == true" - set: field: threat.indicator.type value: ipv4-addr - if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)" + if: "['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)" - rename: field: misp.attribute.value target_field: threat.indicator.url.domain @@ -257,11 +257,11 @@ processors: - set: field: threat.indicator.type value: email-addr - if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)" + if: "['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)" - set: field: threat.indicator.type value: email-message - if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)" + if: "ctx.misp?.attribute?.type?.startsWith('email') == true && !['email-dst', 'email-src'].contains(ctx.misp.attribute.type)" - rename: field: misp.attribute.value target_field: threat.indicator.email.address @@ -274,13 +274,13 @@ processors: - append: field: user.roles value: "reporting_user" - if: ctx?.user?.email != null + if: ctx.user?.email != null ## MAC Address indicator operations - set: field: threat.indicator.type value: mac-addr - if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)" + if: "['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)" - rename: field: misp.attribute.value target_field: threat.indicator.mac @@ -337,12 +337,12 @@ processors: ###################### - remove: field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" ignore_failure: true ignore_missing: true - script: lang: painless - if: ctx?.misp != null + if: ctx.misp != null source: | void handleMap(Map map) { for (def x : map.values()) {