Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[zeek] Make event.original optional #992

Merged
merged 2 commits into from
Jun 8, 2021
+2,951 −1,654
Merged

[zeek] Make event.original optional #992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6269436
Make event.original optional
marc-gr Jun 3, 2021
357d9c5
Change description
marc-gr Jun 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/zeek/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "0.7.3"
changes:
- description: Make event.original optional
type: enhancement
link: https://github.com/elastic/integrations/pull/992
- version: "0.7.2"
changes:
- description: adding back 0.7.0 changes
Expand Down
2 changes: 2 additions & 0 deletions packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,5 @@ dynamic_fields:
event.ingested: ".*"
fields:
"@timestamp": "2020-04-28T11:07:58.223Z"
tags:
- preserve_original_event
47 changes: 35 additions & 12 deletions ...ages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,15 @@
}
},
"event": {
"ingested": "2021-04-23T19:56:22.333182200Z",
"ingested": "2021-06-08T07:47:55.330460300Z",
"original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
"type": "info",
"created": "2020-04-28T11:07:58.223Z",
"kind": "metric"
}
},
"tags": [
"preserve_original_event"
]
},
{
"@timestamp": "2021-03-30T00:04:00.941Z",
Expand All @@ -36,11 +40,15 @@
}
},
"event": {
"ingested": "2021-04-23T19:56:22.333229400Z",
"ingested": "2021-06-08T07:47:55.330466Z",
"original": "{\"ts\":1617062640.941952,\"ts_delta\":900.0005369186401,\"peer\":\"zeek\",\"gaps\":58475,\"acks\":65665,\"percent_lost\":89.05048351481003}",
"type": "info",
"created": "2020-04-28T11:07:58.223Z",
"kind": "metric"
}
},
"tags": [
"preserve_original_event"
]
},
{
"@timestamp": "2021-03-30T00:19:00.942Z",
Expand All @@ -57,11 +65,15 @@
}
},
"event": {
"ingested": "2021-04-23T19:56:22.333237400Z",
"ingested": "2021-06-08T07:47:55.330473300Z",
"original": "{\"ts\":1617063540.942231,\"ts_delta\":900.0002789497376,\"peer\":\"zeek\",\"gaps\":54754,\"acks\":61818,\"percent_lost\":88.5729075673752}",
"type": "info",
"created": "2020-04-28T11:07:58.223Z",
"kind": "metric"
}
},
"tags": [
"preserve_original_event"
]
},
{
"@timestamp": "2021-03-30T00:34:00.942Z",
Expand All @@ -78,11 +90,15 @@
}
},
"event": {
"ingested": "2021-04-23T19:56:22.333243200Z",
"ingested": "2021-06-08T07:47:55.330482600Z",
"original": "{\"ts\":1617064440.942597,\"ts_delta\":900.0003659725189,\"peer\":\"zeek\",\"gaps\":51022,\"acks\":57974,\"percent_lost\":88.00841756649533}",
"type": "info",
"created": "2020-04-28T11:07:58.223Z",
"kind": "metric"
}
},
"tags": [
"preserve_original_event"
]
},
{
"@timestamp": "2021-03-30T00:49:00.942Z",
Expand All @@ -99,11 +115,15 @@
}
},
"event": {
"ingested": "2021-04-23T19:56:22.333248500Z",
"ingested": "2021-06-08T07:47:55.330489900Z",
"original": "{\"ts\":1617065340.942651,\"ts_delta\":900.0000541210175,\"peer\":\"zeek\",\"gaps\":55105,\"acks\":62497,\"percent_lost\":88.17223226714883}",
"type": "info",
"created": "2020-04-28T11:07:58.223Z",
"kind": "metric"
}
},
"tags": [
"preserve_original_event"
]
},
{
"@timestamp": "2019-09-10T16:19:28.465Z",
Expand All @@ -128,12 +148,15 @@
}
},
"event": {
"ingested": "2021-04-23T19:56:22.333253200Z",
"ingested": "2021-06-08T07:47:55.330494Z",
"original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
"type": "info",
"created": "2020-04-28T11:07:58.223Z",
"kind": "metric"
}
},
"tags": [
"preserve_original_event"
]
}
]
}
3 changes: 3 additions & 0 deletions packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,6 @@ tags:
{{#each tags as |tag i|}}
- {{tag}}
{{/each}}
{{#if preserve_original_event}}
- preserve_original_event
{{/if}}
3 changes: 3 additions & 0 deletions packages/zeek/data_stream/capture_loss/agent/stream/log.yml.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ tags:
{{#each tags as |tag i|}}
- {{tag}}
{{/each}}
{{#if preserve_original_event}}
- preserve_original_event
{{/if}}
{{#contains tags "forwarded"}}
publisher_pipeline.disable_host: true
{{/contains}}
61 changes: 37 additions & 24 deletions packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,44 +11,52 @@ processors:
- set:
field: ecs.version
value: '1.9.0'
- json:
- rename:
field: message
target_field: json
ignore_failure: true
if: ctx?.message != null
target_field: event.original
- json:
field: event.original
target_field: _temp_.json
- drop:
if: 'ctx?.json?.result == null && ctx?.json?.ts == null'
- rename:
field: json
target_field: zeek.capture_loss
if: ctx?.json?.ts != null
description: Drop if it is a Splunk event but it is empty.
if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null'
# Splunk specific parsing start
- fingerprint:
fields:
- json.result._cd
- json.result._indextime
- json.result._raw
- json.result._time
- json.result.host
- json.result.source
- _temp_.json.result._cd
- _temp_.json.result._indextime
- _temp_.json.result._raw
- _temp_.json.result._time
- _temp_.json.result.host
- _temp_.json.result.source
target_field: '_id'
if: 'ctx?.json?.result != null && ctx?.json?.ts == null'
- rename:
field: json.result._raw
target_field: event.original
if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null'
- remove:
field: event.original
ignore_missing: true
- json:
if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null'
- set:
field: event.original
target_field: zeek.capture_loss
copy_from: _temp_.json.result._raw
ignore_empty_value: true
ignore_failure: true
if: ctx?.event?.original != null
if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null'
- rename:
field: json.result.host
field: _temp_.json.result.host
target_field: host.name
ignore_missing: true
- rename:
field: json.result.source
field: _temp_.json.result.source
target_field: log.file.path
ignore_missing: true
- remove:
field: _temp_
ignore_missing: true
# Splunk parsing end
- json:
field: event.original
target_field: zeek.capture_loss
ignore_failure: true
- date:
field: zeek.capture_loss.ts
formats:
Expand All @@ -69,6 +77,11 @@ processors:
- message
- json
ignore_missing: true
- remove:
field: event.original
if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
ignore_failure: true
ignore_missing: true
on_failure:
- set:
field: error.message
Expand Down
2 changes: 2 additions & 0 deletions packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,5 @@ dynamic_fields:
event.ingested: ".*"
fields:
"@timestamp": "2020-04-28T11:07:58.223Z"
tags:
- preserve_original_event
Loading