From 692658c9fed090600bfdc067b0a44eff8f6dbe4c Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Mon, 15 Apr 2024 19:12:39 +1000 Subject: [PATCH] [filebeat][threatintel] MISP splitting fix for empty responses (#38917) Two fixes related to empty server responses: - Set `response.split.ignore_empty_value` to `true` to avoid indexing `{response:[]}`, which is the MISP server's empty response body. According to the `response.split` [documentation][1], "If the split target is empty the parent document will be kept. If documents with empty splits should be dropped, the `ignore_empty_value` option should be set to `true`." - Use the null-safe operator for a chained method invocation following a null-safe field access. All other null-safe operator usages were also reviewed and corrected where necessary. [1]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-split --------- Co-authored-by: Dan Kortschak --- CHANGELOG.next.asciidoc | 2 +- .../module/threatintel/misp/config/config.yml | 1 + .../threatintel/misp/ingest/pipeline.yml | 36 +++++++++---------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b58ec371cff..cecdeefd875 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -103,7 +103,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix handling of truncated files in Filestream {issue}38070[38070] {pull}38416[38416] - Fix panic when more than 32767 pipeline clients are active. {issue}38197[38197] {pull}38556[38556] - Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488] - +- [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917] *Heartbeat* diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml index 9ad66efcf54..2f6a2f0e12f 100644 --- a/x-pack/filebeat/module/threatintel/misp/config/config.yml +++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml @@ -49,6 +49,7 @@ request.transforms: response.split: target: body.response + ignore_empty_value: true split: target: body.Event.Attribute ignore_empty_value: true diff --git a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml index 3cdc64d8703..8caba8e738c 100644 --- a/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml @@ -63,12 +63,12 @@ processors: field: - misp.Attribute ignore_missing: true - if: ctx.misp?.Attribute.size() == 0 + if: ctx.misp?.Attribute?.size() == 0 - remove: field: - misp.Object ignore_missing: true - if: ctx.misp?.Object.size() == 0 + if: ctx.misp?.Object?.size() == 0 - date: field: misp.timestamp formats: @@ -144,12 +144,12 @@ processors: - set: field: threat.indicator.type value: file - if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type.startsWith('filename'))" + if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type?.startsWith('filename') == true" - rename: field: misp.attribute.value target_field: "threat.indicator.file.hash.{{misp.attribute.type}}" ignore_missing: true - if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp?.attribute?.type.startsWith('filename')" + if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp.attribute.type.startsWith('filename')" - rename: field: misp.attribute.value target_field: threat.indicator.file.name @@ -160,23 +160,23 @@ processors: patterns: - "%{WORD}\\|%{WORD:_tmp.hashtype}" ignore_missing: true - if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') + if: ctx.misp?.attribute?.type?.startsWith('filename|') == true - grok: field: misp.attribute.value patterns: - "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}" ignore_missing: true - if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') + if: ctx.misp?.attribute?.type?.startsWith('filename|') == true - set: field: threat.indicator.file.hash.{{_tmp.hashtype}} value: "{{_tmp.hashvalue}}" - if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null" + if: "ctx.misp?.attribute?.type?.startsWith('filename|') == true && ctx._tmp?.hashvalue != null && ctx._tmp?.hashtype != null" ## URL/URI indicator operations - set: field: threat.indicator.type value: url - if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)" + if: "['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)" - uri_parts: field: misp.attribute.value target_field: threat.indicator.url @@ -193,7 +193,7 @@ processors: - set: field: threat.indicator.type value: windows-registry-key - if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('regkey')" + if: "ctx.misp?.attribute?.type?.startsWith('regkey') == true" - rename: field: misp.attribute.value target_field: threat.indicator.registry.key @@ -210,7 +210,7 @@ processors: - set: field: threat.indicator.type value: autonomous-system - if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'" + if: "ctx.misp?.attribute?.type == 'AS'" - convert: field: misp.attribute.value type: long @@ -222,11 +222,11 @@ processors: - set: field: threat.indicator.type value: domain-name - if: "ctx.misp?.attribute?.type != null && (ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type.startsWith('domain'))" + if: "ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type?.startsWith('domain') == true" - set: field: threat.indicator.type value: ipv4-addr - if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)" + if: "['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)" - rename: field: misp.attribute.value target_field: threat.indicator.url.domain @@ -257,11 +257,11 @@ processors: - set: field: threat.indicator.type value: email-addr - if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)" + if: "['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)" - set: field: threat.indicator.type value: email-message - if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)" + if: "ctx.misp?.attribute?.type?.startsWith('email') == true && !['email-dst', 'email-src'].contains(ctx.misp.attribute.type)" - rename: field: misp.attribute.value target_field: threat.indicator.email.address @@ -274,13 +274,13 @@ processors: - append: field: user.roles value: "reporting_user" - if: ctx?.user?.email != null + if: ctx.user?.email != null ## MAC Address indicator operations - set: field: threat.indicator.type value: mac-addr - if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)" + if: "['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)" - rename: field: misp.attribute.value target_field: threat.indicator.mac @@ -337,12 +337,12 @@ processors: ###################### - remove: field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" ignore_failure: true ignore_missing: true - script: lang: painless - if: ctx?.misp != null + if: ctx.misp != null source: | void handleMap(Map map) { for (def x : map.values()) {