Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.12](backport #38917) [filebeat][threatintel] MISP splitting fix for empty responses #38927

Merged
merged 2 commits into from
Apr 18, 2024
Merged

[8.12](backport #38917) [filebeat][threatintel] MISP splitting fix for empty responses #38927

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f2e14f8
[filebeat][threatintel] MISP splitting fix for empty responses (#38917)
chrisberkhout Apr 15, 2024
b81446d
Update CHANGELOG.next.asciidoc
chrisberkhout Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]

- Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094]
- Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116]
- [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917]

*Heartbeat*

Expand Down
1 change: 1 addition & 0 deletions x-pack/filebeat/module/threatintel/misp/config/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ request.transforms:

response.split:
target: body.response
ignore_empty_value: true
split:
target: body.Event.Attribute
ignore_empty_value: true
Expand Down
36 changes: 18 additions & 18 deletions x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,12 @@ processors:
field:
- misp.Attribute
ignore_missing: true
if: ctx.misp?.Attribute.size() == 0
if: ctx.misp?.Attribute?.size() == 0
- remove:
field:
- misp.Object
ignore_missing: true
if: ctx.misp?.Object.size() == 0
if: ctx.misp?.Object?.size() == 0
- date:
field: misp.timestamp
formats:
Expand Down Expand Up @@ -144,12 +144,12 @@ processors:
- set:
field: threat.indicator.type
value: file
if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type.startsWith('filename'))"
if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type?.startsWith('filename') == true"
- rename:
field: misp.attribute.value
target_field: "threat.indicator.file.hash.{{misp.attribute.type}}"
ignore_missing: true
if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp?.attribute?.type.startsWith('filename')"
if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp.attribute.type.startsWith('filename')"
- rename:
field: misp.attribute.value
target_field: threat.indicator.file.name
Expand All @@ -160,23 +160,23 @@ processors:
patterns:
- "%{WORD}\\|%{WORD:_tmp.hashtype}"
ignore_missing: true
if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
if: ctx.misp?.attribute?.type?.startsWith('filename|') == true
- grok:
field: misp.attribute.value
patterns:
- "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
ignore_missing: true
if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
if: ctx.misp?.attribute?.type?.startsWith('filename|') == true
- set:
field: threat.indicator.file.hash.{{_tmp.hashtype}}
value: "{{_tmp.hashvalue}}"
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
if: "ctx.misp?.attribute?.type?.startsWith('filename|') == true && ctx._tmp?.hashvalue != null && ctx._tmp?.hashtype != null"

## URL/URI indicator operations
- set:
field: threat.indicator.type
value: url
if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
if: "['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
- uri_parts:
field: misp.attribute.value
target_field: threat.indicator.url
Expand All @@ -193,7 +193,7 @@ processors:
- set:
field: threat.indicator.type
value: windows-registry-key
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('regkey')"
if: "ctx.misp?.attribute?.type?.startsWith('regkey') == true"
- rename:
field: misp.attribute.value
target_field: threat.indicator.registry.key
Expand All @@ -210,7 +210,7 @@ processors:
- set:
field: threat.indicator.type
value: autonomous-system
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'"
if: "ctx.misp?.attribute?.type == 'AS'"
- convert:
field: misp.attribute.value
type: long
Expand All @@ -222,11 +222,11 @@ processors:
- set:
field: threat.indicator.type
value: domain-name
if: "ctx.misp?.attribute?.type != null && (ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type.startsWith('domain'))"
if: "ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type?.startsWith('domain') == true"
- set:
field: threat.indicator.type
value: ipv4-addr
if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
if: "['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.url.domain
Expand Down Expand Up @@ -257,11 +257,11 @@ processors:
- set:
field: threat.indicator.type
value: email-addr
if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
if: "['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
- set:
field: threat.indicator.type
value: email-message
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
if: "ctx.misp?.attribute?.type?.startsWith('email') == true && !['email-dst', 'email-src'].contains(ctx.misp.attribute.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.email.address
Expand All @@ -274,13 +274,13 @@ processors:
- append:
field: user.roles
value: "reporting_user"
if: ctx?.user?.email != null
if: ctx.user?.email != null

## MAC Address indicator operations
- set:
field: threat.indicator.type
value: mac-addr
if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
if: "['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.mac
Expand Down Expand Up @@ -337,12 +337,12 @@ processors:
######################
- remove:
field: event.original
if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
ignore_failure: true
ignore_missing: true
- script:
lang: painless
if: ctx?.misp != null
if: ctx.misp != null
source: |
void handleMap(Map map) {
for (def x : map.values()) {
Expand Down
Loading