diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml index 19ecfc8d9ea..75241aff764 100644 --- a/packages/zeek/changelog.yml +++ b/packages/zeek/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.7.3" + changes: + - description: Make event.original optional + type: enhancement + link: https://github.com/elastic/integrations/pull/992 - version: "0.7.2" changes: - description: adding back 0.7.0 changes diff --git a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-config.yml b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-config.yml +++ b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json index 8cb72b48bc7..441b5c4b5cc 100644 --- a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json +++ b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json @@ -15,11 +15,15 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333182200Z", + "ingested": "2021-06-08T07:47:55.330460300Z", + "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2021-03-30T00:04:00.941Z", @@ -36,11 +40,15 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333229400Z", + "ingested": "2021-06-08T07:47:55.330466Z", + "original": "{\"ts\":1617062640.941952,\"ts_delta\":900.0005369186401,\"peer\":\"zeek\",\"gaps\":58475,\"acks\":65665,\"percent_lost\":89.05048351481003}", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2021-03-30T00:19:00.942Z", @@ -57,11 +65,15 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333237400Z", + "ingested": "2021-06-08T07:47:55.330473300Z", + "original": "{\"ts\":1617063540.942231,\"ts_delta\":900.0002789497376,\"peer\":\"zeek\",\"gaps\":54754,\"acks\":61818,\"percent_lost\":88.5729075673752}", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2021-03-30T00:34:00.942Z", @@ -78,11 +90,15 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333243200Z", + "ingested": "2021-06-08T07:47:55.330482600Z", + "original": "{\"ts\":1617064440.942597,\"ts_delta\":900.0003659725189,\"peer\":\"zeek\",\"gaps\":51022,\"acks\":57974,\"percent_lost\":88.00841756649533}", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2021-03-30T00:49:00.942Z", @@ -99,11 +115,15 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333248500Z", + "ingested": "2021-06-08T07:47:55.330489900Z", + "original": "{\"ts\":1617065340.942651,\"ts_delta\":900.0000541210175,\"peer\":\"zeek\",\"gaps\":55105,\"acks\":62497,\"percent_lost\":88.17223226714883}", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2019-09-10T16:19:28.465Z", @@ -128,12 +148,15 @@ } }, "event": { - "ingested": "2021-04-23T19:56:22.333253200Z", + "ingested": "2021-06-08T07:47:55.330494Z", "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}", "type": "info", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/capture_loss/agent/stream/log.yml.hbs b/packages/zeek/data_stream/capture_loss/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/capture_loss/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/capture_loss/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml index b09928978e0..92975d5a3eb 100644 --- a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml @@ -11,44 +11,52 @@ processors: - set: field: ecs.version value: '1.9.0' - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.capture_loss - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true - - json: + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: field: event.original - target_field: zeek.capture_loss + copy_from: _temp_.json.result._raw + ignore_empty_value: true ignore_failure: true - if: ctx?.event?.original != null + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end + - json: + field: event.original + target_field: zeek.capture_loss + ignore_failure: true - date: field: zeek.capture_loss.ts formats: @@ -69,6 +77,11 @@ processors: - message - json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-config.yml b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-config.yml +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json index d7e342d0c76..e7974c426d8 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json @@ -39,7 +39,8 @@ }, "event": { "duration": 76967000, - "ingested": "2021-04-23T19:56:22.481489300Z", + "ingested": "2021-06-08T07:50:09.231322300Z", + "original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":true,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CAcJw21BbVedgFnYH3", @@ -50,6 +51,11 @@ "end" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:Z26DBGVYoBKQ1FT6qfPaAqBnJik=", @@ -57,11 +63,7 @@ "bytes": 309, "packets": 2, "direction": "internal" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2019-01-11T06:33:36.857Z", @@ -117,7 +119,8 @@ }, "event": { "duration": 76967000, - "ingested": "2021-04-23T19:56:22.481509800Z", + "ingested": "2021-06-08T07:50:09.231333400Z", + "original": "{\"ts\":1547188416.857497,\"uid\":\"CAcJw21BbVedgFnYH4\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38340,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CAcJw21BbVedgFnYH4", @@ -128,6 +131,11 @@ "end" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:77KJyeznYjdDxCSKdZhW89aAaBI=", @@ -135,11 +143,7 @@ "bytes": 309, "packets": 2, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2019-01-11T06:33:37.857Z", @@ -210,7 +214,8 @@ }, "event": { "duration": 76967000, - "ingested": "2021-04-23T19:56:22.481515200Z", + "ingested": "2021-06-08T07:50:09.231341500Z", + "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CAcJw21BbVedgFnYH5", @@ -221,6 +226,11 @@ "end" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:hWC6cnCoeyQehzquxJQU6Y3Wm3g=", @@ -228,11 +238,7 @@ "bytes": 309, "packets": 2, "direction": "external" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2019-03-01T00:10:00.578Z", @@ -273,27 +279,29 @@ "ip": "192.0.2.205" }, "event": { - "ingested": "2021-04-23T19:56:22.481522400Z", + "ingested": "2021-06-08T07:50:09.231349500Z", + "original": "{\"ts\":1551399000.57855,\"uid\":\"Cc6NJ3GRlfjE44I3h\",\"id.orig_h\":\"192.0.2.205\",\"id.orig_p\":3,\"id.resp_h\":\"198.51.100.249\",\"id.resp_p\":3,\"proto\":\"icmp\",\"conn_state\":\"OTH\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"orig_pkts\":1,\"orig_ip_bytes\":107,\"resp_pkts\":0,\"resp_ip_bytes\":0,\"tunnel_parents\":[]}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "Cc6NJ3GRlfjE44I3h", "category": "network", "type": [ "connection", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "community_id": "1:gzTID87+KHoT4RFDSqb5aInTPeg=", "transport": "icmp", "bytes": 107, "packets": 1, "direction": "external" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-30T00:00:00.404Z", @@ -351,27 +359,29 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:22.481526700Z", + "ingested": "2021-06-08T07:50:09.231357300Z", + "original": "{\"ts\":1617062400.404645,\"uid\":\"CCicIg43lOtCQOxXnb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":56190,\"id.resp_h\":\"46.101.87.151\",\"id.resp_p\":443,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CCicIg43lOtCQOxXnb", "category": "network", "type": [ "connection", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "community_id": "1:ziCfaAfpSmrkSIWraOMW2mxUmFc=", "transport": "tcp", "bytes": 0, "packets": 0, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-29T23:55:00.419Z", @@ -430,7 +440,8 @@ }, "event": { "duration": 103708982, - "ingested": "2021-04-23T19:56:22.481531Z", + "ingested": "2021-06-08T07:50:09.231365600Z", + "original": "{\"ts\":1617062100.419397,\"uid\":\"C52mXBCPJ4pPGkhr1\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60810,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10370898246765137,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "C52mXBCPJ4pPGkhr1", @@ -440,17 +451,18 @@ "info" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "community_id": "1:c8VbaUJYZDhCA0Us2hi3JYTahPI=", "transport": "tcp", "bytes": 319, "packets": 5, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-29T23:55:00.419Z", @@ -509,7 +521,8 @@ }, "event": { "duration": 104128838, - "ingested": "2021-04-23T19:56:22.481535Z", + "ingested": "2021-06-08T07:50:09.231373500Z", + "original": "{\"ts\":1617062100.419603,\"uid\":\"CTzCky2CyLT5JJvHck\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60804,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10412883758544922,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CTzCky2CyLT5JJvHck", @@ -519,17 +532,18 @@ "info" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "community_id": "1:8EPi737PZXW0ZMOuEpsZ0CWS+UY=", "transport": "tcp", "bytes": 319, "packets": 5, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-29T23:55:00.419Z", @@ -588,7 +602,8 @@ }, "event": { "duration": 104333878, - "ingested": "2021-04-23T19:56:22.481550900Z", + "ingested": "2021-06-08T07:50:09.231381400Z", + "original": "{\"ts\":1617062100.419826,\"uid\":\"CIkS28PDxqQnN49m2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60802,\"id.resp_h\":\"20.190.160.73\",\"id.resp_p\":443,\"proto\":\"tcp\",\"duration\":0.10433387756347656,\"orig_bytes\":0,\"resp_bytes\":5854,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"^hCcdafA\",\"orig_pkts\":1,\"orig_ip_bytes\":52,\"resp_pkts\":4,\"resp_ip_bytes\":267}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CIkS28PDxqQnN49m2", @@ -598,17 +613,18 @@ "info" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "community_id": "1:D/bWvCWz34T0lAiafMBSMauT08c=", "transport": "tcp", "bytes": 319, "packets": 5, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-29T23:59:50.563Z", @@ -649,7 +665,8 @@ }, "event": { "duration": 26802063, - "ingested": "2021-04-23T19:56:22.481559200Z", + "ingested": "2021-06-08T07:50:09.231389200Z", + "original": "{\"ts\":1617062390.563187,\"uid\":\"CezEGe4jeLNkayV976\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":38948,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.02680206298828125,\"orig_bytes\":0,\"resp_bytes\":241,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":269}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CezEGe4jeLNkayV976", @@ -659,6 +676,11 @@ "info" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:MaT7kz/SkupGvuFVoZ2W3Q8LBfo=", @@ -666,11 +688,7 @@ "bytes": 269, "packets": 1, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-29T23:59:50.563Z", @@ -711,7 +729,8 @@ }, "event": { "duration": 25056124, - "ingested": "2021-04-23T19:56:22.481564200Z", + "ingested": "2021-06-08T07:50:09.231397200Z", + "original": "{\"ts\":1617062390.563442,\"uid\":\"CKSr3w18mmW6t7bXC4\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":40080,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.025056123733520509,\"orig_bytes\":0,\"resp_bytes\":276,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":304}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CKSr3w18mmW6t7bXC4", @@ -721,6 +740,11 @@ "info" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:UdmcTRiLwwI31qutgJjuqtMRZOE=", @@ -728,11 +752,7 @@ "bytes": 304, "packets": 1, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-29T23:59:50.667Z", @@ -773,7 +793,8 @@ }, "event": { "duration": 3319979, - "ingested": "2021-04-23T19:56:22.481568400Z", + "ingested": "2021-06-08T07:50:09.231401300Z", + "original": "{\"ts\":1617062390.667048,\"uid\":\"CGUiHy4kLIF2ml95eg\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":41407,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.003319978713989258,\"orig_bytes\":0,\"resp_bytes\":133,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":161}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CGUiHy4kLIF2ml95eg", @@ -783,6 +804,11 @@ "info" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:bCnXQXEPKVtzbvpzqWsri1DRNpc=", @@ -790,11 +816,7 @@ "bytes": 161, "packets": 1, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-29T23:59:50.698Z", @@ -835,7 +857,8 @@ }, "event": { "duration": 1111984, - "ingested": "2021-04-23T19:56:22.481572700Z", + "ingested": "2021-06-08T07:50:09.231407200Z", + "original": "{\"ts\":1617062390.698943,\"uid\":\"CAOZZi4Qrio7gUVgVc\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":50487,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0011119842529296876,\"orig_bytes\":0,\"resp_bytes\":202,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":230}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CAOZZi4Qrio7gUVgVc", @@ -845,6 +868,11 @@ "info" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:tv5zJqq58ufw3fbvDlETbMFY800=", @@ -852,11 +880,7 @@ "bytes": 230, "packets": 1, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-29T23:59:50.699Z", @@ -897,7 +921,8 @@ }, "event": { "duration": 908852, - "ingested": "2021-04-23T19:56:22.481576800Z", + "ingested": "2021-06-08T07:50:09.231434200Z", + "original": "{\"ts\":1617062390.699227,\"uid\":\"Chx5fs3xQ5ALB72i4e\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":49647,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.0009088516235351563,\"orig_bytes\":0,\"resp_bytes\":145,\"conn_state\":\"SHR\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Cd\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":1,\"resp_ip_bytes\":173}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "Chx5fs3xQ5ALB72i4e", @@ -907,6 +932,11 @@ "info" ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:B3k4XMlSCTUFWj04Y3MgyIdpqV0=", @@ -914,11 +944,7 @@ "bytes": 173, "packets": 1, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-30T00:00:00.703Z", @@ -958,27 +984,29 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:22.481581200Z", + "ingested": "2021-06-08T07:50:09.231441100Z", + "original": "{\"ts\":1617062400.703865,\"uid\":\"C3pPjh1YRYcVDiZD3\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44944,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C3pPjh1YRYcVDiZD3", "category": "network", "type": [ "connection", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "community_id": "1:zlvv5I2WDugh9zCIwgFKRCr6aFs=", "transport": "tcp", "bytes": 0, "packets": 0, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-30T00:00:00.703Z", @@ -1018,27 +1046,29 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:22.481584800Z", + "ingested": "2021-06-08T07:50:09.231444900Z", + "original": "{\"ts\":1617062400.703851,\"uid\":\"ChUxTmYLG37oO5qUb\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44942,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "ChUxTmYLG37oO5qUb", "category": "network", "type": [ "connection", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "community_id": "1:E6r4npj9JGrYl1AJYRB9WsOSuq4=", "transport": "tcp", "bytes": 0, "packets": 0, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "@timestamp": "2021-03-30T00:00:00.704Z", @@ -1078,27 +1108,29 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:22.481588500Z", + "ingested": "2021-06-08T07:50:09.231449Z", + "original": "{\"ts\":1617062400.704467,\"uid\":\"CpeAOT3B11CTXJgzw2\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44946,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":80,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"C\",\"orig_pkts\":0,\"orig_ip_bytes\":0,\"resp_pkts\":0,\"resp_ip_bytes\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CpeAOT3B11CTXJgzw2", "category": "network", "type": [ "connection", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "community_id": "1:P1p7mEvXXmc+vW6oKL9TO8XgiOQ=", "transport": "tcp", "bytes": 0, "packets": 0, "direction": "outbound" - }, - "tags": [ - "local_orig", - "local_resp" - ] + } }, { "log": { @@ -1162,6 +1194,11 @@ "ip": "4.4.2.2", "packets": 1 }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], "network": { "protocol": "dns", "community_id": "1:hWC6cnCoeyQehzquxJQU6Y3Wm3g=", @@ -1170,10 +1207,6 @@ "packets": 2, "direction": "external" }, - "tags": [ - "local_orig", - "local_resp" - ], "@timestamp": "2019-01-11T06:33:37.857Z", "ecs": { "version": "1.9.0" @@ -1189,7 +1222,7 @@ }, "event": { "duration": 76967000, - "ingested": "2021-04-23T19:56:22.481592400Z", + "ingested": "2021-06-08T07:50:09.231454200Z", "original": "{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"4.4.2.2\",\"id.orig_p\":38334,\"id.resp_h\":\"8.8.8.8\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/connection/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/connection/agent/stream/log.yml.hbs b/packages/zeek/data_stream/connection/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/connection/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/connection/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml index abeac41bf3c..256078614fb 100644 --- a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml @@ -17,44 +17,52 @@ processors: - set: field: event.category value: network - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.connection - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.connection ignore_failure: true - if: ctx?.event?.original != null - dot_expander: path: zeek.connection field: id.orig_p @@ -350,6 +358,11 @@ processors: - json - temp ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-config.yml b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-config.yml +++ b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json index 7f811dd1f82..f887e2af937 100644 --- a/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json +++ b/packages/zeek/data_stream/dce_rpc/_dev/test/pipeline/test-dce-rpc.log-expected.json @@ -31,7 +31,8 @@ "ip": "172.16.133.6" }, "event": { - "ingested": "2021-04-23T19:56:23.054640400Z", + "ingested": "2021-06-08T07:52:23.769834400Z", + "original": "{\"ts\":1361916332.298338,\"uid\":\"CsNHVHa1lzFtvJzT8\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"rtt\":0.09211,\"named_pipe\":\"\\u005cPIPE\\u005cbrowser\",\"endpoint\":\"browser\",\"operation\":\"BrowserrQueryOtherDomains\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "BrowserrQueryOtherDomains", @@ -45,6 +46,9 @@ "info" ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "dce_rpc", "community_id": "1:SJNAD5vtzZuhQjGtfaI8svTnyuw=", @@ -76,6 +80,9 @@ "address": "172.16.133.6", "ip": "172.16.133.6" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "dce_rpc", "community_id": "1:SJNAD5vtzZuhQjGtfaI8svTnyuw=", @@ -95,7 +102,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:23.054658400Z", + "ingested": "2021-06-08T07:52:23.769843100Z", "original": "{\"ts\":1361916332.298338,\"uid\":\"CsNHVHa1lzFtvJzT8\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"rtt\":0.09211,\"named_pipe\":\"\\u005cPIPE\\u005cbrowser\",\"endpoint\":\"browser\",\"operation\":\"BrowserrQueryOtherDomains\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dce_rpc/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/dce_rpc/agent/stream/log.yml.hbs b/packages/zeek/data_stream/dce_rpc/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/dce_rpc/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/dce_rpc/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml index 44572c149c7..8d230f46bfd 100644 --- a/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dce_rpc/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: dce_rpc - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.dce_rpc - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.dce_rpc @@ -184,6 +193,11 @@ processors: - json - zeek.dce_rpc.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-config.yml b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-config.yml +++ b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json index 202f7e118e1..09ce2b6c3d5 100644 --- a/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json +++ b/packages/zeek/data_stream/dhcp/_dev/test/pipeline/test-dhcp.log-expected.json @@ -40,6 +40,9 @@ "address": "192.168.199.132", "ip": "192.168.199.132" }, + "tags": [ + "preserve_original_event" + ], "network": { "name": "localdomain", "protocol": "dhcp", @@ -60,7 +63,10 @@ "address": "192.168.199.132" }, "event": { - "ingested": "2021-04-23T19:56:23.150661700Z", + "ingested": "2021-06-08T07:54:57.784810900Z", + "original": "{\"ts\":1476605498.771847,\"uids\":[\"CmWOt6VWaNGqXYcH6\",\"CLObLo4YHn0u23Tp8a\"],\"client_addr\":\"192.168.199.132\",\"server_addr\":\"192.168.199.254\",\"mac\":\"00:0c:29:03:df:ad\",\"host_name\":\"DESKTOP-2AEFM7G\",\"client_fqdn\":\"DESKTOP-2AEFM7G\",\"domain\":\"localdomain\",\"requested_addr\":\"192.168.199.132\",\"assigned_addr\":\"192.168.199.132\",\"lease_time\":1800.0,\"msg_types\":[\"REQUEST\",\"ACK\"],\"duration\":0.000161}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": [ "CmWOt6VWaNGqXYcH6", "CLObLo4YHn0u23Tp8a" @@ -72,9 +78,7 @@ "connection", "protocol", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -112,6 +116,9 @@ "address": "10.156.0.2", "ip": "10.156.0.2" }, + "tags": [ + "preserve_original_event" + ], "network": { "name": "c.elastic-sa.internal", "protocol": "dhcp", @@ -132,7 +139,10 @@ "address": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.150679900Z", + "ingested": "2021-06-08T07:54:57.784822600Z", + "original": "{\"ts\":1617088722.072416,\"uids\":[\"Ck0tsG4wsJxI3lIEZ\"],\"client_addr\":\"10.156.0.2\",\"server_addr\":\"169.254.169.254\",\"mac\":\"42:01:0a:9c:00:02\",\"domain\":\"c.elastic-sa.internal\",\"assigned_addr\":\"10.156.0.2\",\"lease_time\":86400.0,\"msg_types\":[\"ACK\"],\"duration\":0.0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": [ "Ck0tsG4wsJxI3lIEZ" ], @@ -143,9 +153,7 @@ "connection", "protocol", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -193,6 +201,9 @@ "address": "192.168.199.132", "ip": "192.168.199.132" }, + "tags": [ + "preserve_original_event" + ], "network": { "name": "localdomain", "protocol": "dhcp", @@ -216,7 +227,7 @@ "address": "192.168.199.132" }, "event": { - "ingested": "2021-04-23T19:56:23.150685Z", + "ingested": "2021-06-08T07:54:57.784831200Z", "original": "{\"ts\":1476605498.771847,\"uids\":[\"CmWOt6VWaNGqXYcH6\",\"CLObLo4YHn0u23Tp8a\"],\"client_addr\":\"192.168.199.132\",\"server_addr\":\"192.168.199.254\",\"mac\":\"00:0c:29:03:df:ad\",\"host_name\":\"DESKTOP-2AEFM7G\",\"client_fqdn\":\"DESKTOP-2AEFM7G\",\"domain\":\"localdomain\",\"requested_addr\":\"192.168.199.132\",\"assigned_addr\":\"192.168.199.132\",\"lease_time\":1800.0,\"msg_types\":[\"REQUEST\",\"ACK\"],\"duration\":0.000161}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dhcp/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/dhcp/agent/stream/log.yml.hbs b/packages/zeek/data_stream/dhcp/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/dhcp/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/dhcp/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml index 2040ba98822..fbbe5083ad6 100644 --- a/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dhcp/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: dhcp - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.dhcp - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.dhcp @@ -205,6 +214,11 @@ processors: - message - json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-config.yml b/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-config.yml +++ b/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json b/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json index 8a0d8a3d6ba..4b3e5d764b3 100644 --- a/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json +++ b/packages/zeek/data_stream/dnp3/_dev/test/pipeline/test-dnp3.log-expected.json @@ -29,7 +29,8 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-04-23T19:56:23.292654100Z", + "ingested": "2021-06-08T07:55:37.596665Z", + "original": "{\"ts\":1227729908.705944,\"uid\":\"CQV6tj1w1t4WzQpHoe\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":42942,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":20000,\"fc_request\":\"READ\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "read", @@ -43,6 +44,9 @@ "info" ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "dnp3", "community_id": "1:E57Z1w3RrSdR+fi6rSZblbQVhzY=", @@ -73,6 +77,9 @@ "address": "127.0.0.1", "ip": "127.0.0.1" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "dnp3", "community_id": "1:E57Z1w3RrSdR+fi6rSZblbQVhzY=", @@ -91,7 +98,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:23.292667900Z", + "ingested": "2021-06-08T07:55:37.596678500Z", "original": "{\"ts\":1227729908.705944,\"uid\":\"CQV6tj1w1t4WzQpHoe\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":42942,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":20000,\"fc_request\":\"READ\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dnp3/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/dnp3/agent/stream/log.yml.hbs b/packages/zeek/data_stream/dnp3/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/dnp3/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/dnp3/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml index fee837ebb9d..d60fea17588 100644 --- a/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dnp3/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: dnp3 - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.dnp3 - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.dnp3 @@ -203,6 +212,11 @@ processors: - json - zeek.dnp3.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-config.yml b/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-config.yml +++ b/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json b/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json index eef13847171..fb94b46ca60 100644 --- a/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json +++ b/packages/zeek/data_stream/dns/_dev/test/pipeline/test-dns.log-expected.json @@ -1,16 +1,6 @@ { "expected": [ { - "@timestamp": "2019-01-11T06:33:35.857Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "192.168.86.167", - "192.168.86.1" - ] - }, "destination": { "port": 53, "address": "192.168.86.1", @@ -84,9 +74,27 @@ "address": "192.168.86.167", "ip": "192.168.86.167" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "community_id": "1:Z26DBGVYoBKQ1FT6qfPaAqBnJik=", + "transport": "udp" + }, + "@timestamp": "2019-01-11T06:33:35.857Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "192.168.86.167", + "192.168.86.1" + ] + }, "event": { "duration": 7.6967E7, - "ingested": "2021-04-23T19:56:23.442358600Z", + "ingested": "2021-06-03T13:43:27.545545646Z", "original": "{\"ts\":1547188415.857497,\"uid\":\"CAcJw21BbVedgFnYH3\",\"id.orig_h\":\"192.168.86.167\",\"id.orig_p\":38339,\"id.resp_h\":\"192.168.86.1\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":15209,\"rtt\":0.076967,\"query\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":1,\"qtype_name\":\"A\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":true,\"RA\":true,\"Z\":0,\"answers\":[\"proxy-production-us-west1.gcp.cloud.es.io\",\"proxy-production-us-west1-v1-009.gcp.cloud.es.io\",\"35.199.178.4\"],\"TTLs\":[119.0,119.0,59.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -100,24 +108,9 @@ "info" ], "outcome": "success" - }, - "network": { - "protocol": "dns", - "community_id": "1:Z26DBGVYoBKQ1FT6qfPaAqBnJik=", - "transport": "udp" } }, { - "@timestamp": "2019-08-29T16:23:50.680Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "fe80::4ef:15cf:769f:ff21", - "ff02::fb" - ] - }, "destination": { "port": 5353, "address": "ff02::fb", @@ -152,8 +145,26 @@ "address": "fe80::4ef:15cf:769f:ff21", "ip": "fe80::4ef:15cf:769f:ff21" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "community_id": "1:Jq0sRtlGSMjsvMBE1ZYybbR2tI0=", + "transport": "udp" + }, + "@timestamp": "2019-08-29T16:23:50.680Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "fe80::4ef:15cf:769f:ff21", + "ff02::fb" + ] + }, "event": { - "ingested": "2021-04-23T19:56:23.442375900Z", + "ingested": "2021-06-03T13:43:27.545551315Z", "original": "{\"ts\":1567095830.680046,\"uid\":\"C19a1k4lTv46YMbeOk\",\"id.orig_h\":\"fe80::4ef:15cf:769f:ff21\",\"id.orig_p\":5353,\"id.resp_h\":\"ff02::fb\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"qclass\":1,\"qclass_name\":\"C_INTERNET\",\"qtype\":12,\"qtype_name\":\"PTR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -167,24 +178,9 @@ "info" ], "outcome": "success" - }, - "network": { - "protocol": "dns", - "community_id": "1:Jq0sRtlGSMjsvMBE1ZYybbR2tI0=", - "transport": "udp" } }, { - "@timestamp": "2019-08-29T16:23:50.734Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "192.168.86.237", - "224.0.0.251" - ] - }, "destination": { "port": 5353, "address": "224.0.0.251", @@ -230,8 +226,26 @@ "address": "192.168.86.237", "ip": "192.168.86.237" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "community_id": "1:QIR5YXlirWwWA18ZyY/RnvQoaic=", + "transport": "udp" + }, + "@timestamp": "2019-08-29T16:23:50.734Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "192.168.86.237", + "224.0.0.251" + ] + }, "event": { - "ingested": "2021-04-23T19:56:23.442399400Z", + "ingested": "2021-06-03T13:43:27.545553489Z", "original": "{\"ts\":1567095830.734329,\"uid\":\"CdiVAw7jJw6gsX5H\",\"id.orig_h\":\"192.168.86.237\",\"id.orig_p\":5353,\"id.resp_h\":\"224.0.0.251\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":true,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"answers\":[\"bravia-4k-gb-5c89f865c9d569ab338815b35e3acc56._googlecast._tcp.local\"],\"TTLs\":[120.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -245,24 +259,9 @@ "info" ], "outcome": "success" - }, - "network": { - "protocol": "dns", - "community_id": "1:QIR5YXlirWwWA18ZyY/RnvQoaic=", - "transport": "udp" } }, { - "@timestamp": "2021-03-30T11:59:52.091Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "169.254.169.254" - ] - }, "destination": { "port": 53, "address": "169.254.169.254", @@ -328,8 +327,26 @@ "address": "10.156.0.2", "ip": "10.156.0.2" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "community_id": "1:Mj0uP/7Ctd+meHQL8iXVrCNL2ZE=", + "transport": "udp" + }, + "@timestamp": "2021-03-30T11:59:52.091Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "169.254.169.254" + ] + }, "event": { - "ingested": "2021-04-23T19:56:23.442407200Z", + "ingested": "2021-06-03T13:43:27.545555447Z", "original": "{\"ts\":1617105592.091052,\"uid\":\"CpwXdW4LQaJkaIgpk\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":33438,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":58036,\"query\":\"manage.office.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"manage.office.com.trafficmanager.net\",\"o365adtapiproddeu001.cloudapp.net\",\"51.116.158.62\"],\"TTLs\":[13.0,18.0,8.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -343,24 +360,9 @@ "info" ], "outcome": "success" - }, - "network": { - "protocol": "dns", - "community_id": "1:Mj0uP/7Ctd+meHQL8iXVrCNL2ZE=", - "transport": "udp" } }, { - "@timestamp": "2021-03-30T11:59:52.973Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "169.254.169.254" - ] - }, "destination": { "port": 53, "address": "169.254.169.254", @@ -424,8 +426,26 @@ "address": "10.156.0.2", "ip": "10.156.0.2" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "community_id": "1:0B1VNLwfmVgcZUY1gi6ZVuS8YZE=", + "transport": "udp" + }, + "@timestamp": "2021-03-30T11:59:52.973Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "169.254.169.254" + ] + }, "event": { - "ingested": "2021-04-23T19:56:23.442412Z", + "ingested": "2021-06-03T13:43:27.545557324Z", "original": "{\"ts\":1617105592.973919,\"uid\":\"CO5TE748RoJEZuOThl\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":60444,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":35744,\"query\":\"login.microsoftonline.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"a.privatelink.msidentity.com\",\"prda.aadg.msidentity.com\",\"www.tm.a.prd.aadg.akadns.net\"],\"TTLs\":[296.0,287.0,287.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -439,24 +459,9 @@ "info" ], "outcome": "success" - }, - "network": { - "protocol": "dns", - "community_id": "1:0B1VNLwfmVgcZUY1gi6ZVuS8YZE=", - "transport": "udp" } }, { - "@timestamp": "2021-03-30T11:59:52.974Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "169.254.169.254" - ] - }, "destination": { "port": 53, "address": "169.254.169.254", @@ -577,8 +582,26 @@ "address": "10.156.0.2", "ip": "10.156.0.2" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "community_id": "1:6FS8lMU9Y2cS38F7kmqpZmgcpbs=", + "transport": "udp" + }, + "@timestamp": "2021-03-30T11:59:52.974Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "169.254.169.254" + ] + }, "event": { - "ingested": "2021-04-23T19:56:23.442415600Z", + "ingested": "2021-06-03T13:43:27.545559117Z", "original": "{\"ts\":1617105592.9742,\"uid\":\"CG1jsmeHcBCGnWXmk\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":44310,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":58458,\"query\":\"login.microsoftonline.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"a.privatelink.msidentity.com\",\"prda.aadg.msidentity.com\",\"www.tm.a.prd.aadg.trafficmanager.net\",\"20.190.159.132\",\"40.126.31.143\",\"20.190.159.134\",\"40.126.31.1\",\"20.190.159.136\",\"40.126.31.135\",\"40.126.31.6\",\"20.190.159.138\"],\"TTLs\":[299.0,214.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0,243.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -592,24 +615,9 @@ "info" ], "outcome": "success" - }, - "network": { - "protocol": "dns", - "community_id": "1:6FS8lMU9Y2cS38F7kmqpZmgcpbs=", - "transport": "udp" } }, { - "@timestamp": "2021-03-30T11:59:53.106Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "169.254.169.254" - ] - }, "destination": { "port": 53, "address": "169.254.169.254", @@ -675,8 +683,26 @@ "address": "10.156.0.2", "ip": "10.156.0.2" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "dns", + "community_id": "1:o8PIGtc58C2kli9WTnYzRHbKTwM=", + "transport": "udp" + }, + "@timestamp": "2021-03-30T11:59:53.106Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "169.254.169.254" + ] + }, "event": { - "ingested": "2021-04-23T19:56:23.442421100Z", + "ingested": "2021-06-03T13:43:27.545560880Z", "original": "{\"ts\":1617105593.106256,\"uid\":\"ChP0cl4j5mbXSZ9TGf\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":36364,\"id.resp_h\":\"169.254.169.254\",\"id.resp_p\":53,\"proto\":\"udp\",\"trans_id\":8791,\"query\":\"manage.office.com\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":false,\"TC\":false,\"RD\":false,\"RA\":true,\"Z\":0,\"answers\":[\"manage.office.com.trafficmanager.net\",\"o365adtapiproddeu001.cloudapp.net\",\"51.116.158.62\"],\"TTLs\":[12.0,17.0,7.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", @@ -690,11 +716,6 @@ "info" ], "outcome": "success" - }, - "network": { - "protocol": "dns", - "community_id": "1:o8PIGtc58C2kli9WTnYzRHbKTwM=", - "transport": "udp" } }, { @@ -748,6 +769,9 @@ "address": "192.168.86.237", "ip": "192.168.86.237" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "dns", "community_id": "1:QIR5YXlirWwWA18ZyY/RnvQoaic=", @@ -767,7 +791,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:23.442424900Z", + "ingested": "2021-06-03T13:43:27.545562714Z", "original": "{\"ts\":1567095830.734329,\"uid\":\"CdiVAw7jJw6gsX5H\",\"id.orig_h\":\"192.168.86.237\",\"id.orig_p\":5353,\"id.resp_h\":\"224.0.0.251\",\"id.resp_p\":5353,\"proto\":\"udp\",\"trans_id\":0,\"query\":\"_googlecast._tcp.local\",\"rcode\":0,\"rcode_name\":\"NOERROR\",\"AA\":true,\"TC\":false,\"RD\":false,\"RA\":false,\"Z\":0,\"answers\":[\"bravia-4k-gb-5c89f865c9d569ab338815b35e3acc56._googlecast._tcp.local\"],\"TTLs\":[120.0],\"rejected\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dns/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/dns/agent/stream/log.yml.hbs b/packages/zeek/data_stream/dns/agent/stream/log.yml.hbs index 3b5aad8c8ea..94c217fe0f0 100644 --- a/packages/zeek/data_stream/dns/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/dns/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} \ No newline at end of file diff --git a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml index b08ade877ea..d0af1a3b34d 100644 --- a/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dns/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: dns - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.dns - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.dns @@ -315,6 +324,11 @@ processors: - zeek.dns.id - message ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-config.yml b/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-config.yml +++ b/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json b/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json index 6a64634b808..5088497ffd6 100644 --- a/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json +++ b/packages/zeek/data_stream/dpd/_dev/test/pipeline/test-dpd.log-expected.json @@ -29,7 +29,10 @@ "ip": "192.168.10.31" }, "event": { - "ingested": "2021-04-23T19:56:23.750033600Z", + "ingested": "2021-06-08T07:57:48.385311900Z", + "original": "{\"ts\":1507567500.423033,\"uid\":\"CRrT7S1ccw9H6hzCR\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49285,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":445,\"proto\":\"tcp\",\"analyzer\":\"DCE_RPC\",\"failure_reason\":\"Binpac exception: binpac exception: \\u0026enforce violation : DCE_RPC_Header:rpc_vers\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CRrT7S1ccw9H6hzCR", "category": [ "network" @@ -37,10 +40,11 @@ "type": [ "connection", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:b+Szw+ia464igf5e+MwW1WUzw9Y=", "transport": "tcp" @@ -69,6 +73,9 @@ "address": "192.168.10.31", "ip": "192.168.10.31" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:b+Szw+ia464igf5e+MwW1WUzw9Y=", "transport": "tcp" @@ -87,7 +94,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:23.750047800Z", + "ingested": "2021-06-08T07:57:48.385315700Z", "original": "{\"ts\":1507567500.423033,\"uid\":\"CRrT7S1ccw9H6hzCR\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49285,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":445,\"proto\":\"tcp\",\"analyzer\":\"DCE_RPC\",\"failure_reason\":\"Binpac exception: binpac exception: \\u0026enforce violation : DCE_RPC_Header:rpc_vers\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/dpd/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/dpd/agent/stream/log.yml.hbs b/packages/zeek/data_stream/dpd/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/dpd/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/dpd/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml index aafe4356c67..c0504cc531d 100644 --- a/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/dpd/elasticsearch/ingest_pipeline/default.yml @@ -23,39 +23,48 @@ processors: - append: field: event.type value: info - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.dpd - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.dpd @@ -177,6 +186,11 @@ processors: - json - zeek.dpd.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-config.yml b/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-config.yml +++ b/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json b/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json index 53538a9879c..99dfa10a603 100644 --- a/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json +++ b/packages/zeek/data_stream/files/_dev/test/pipeline/test-files.log-expected.json @@ -57,17 +57,21 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-04-23T19:56:23.863999500Z", + "ingested": "2021-06-08T07:58:45.531440800Z", + "original": "{\"ts\":1547688796.636812,\"fuid\":\"FMkioa222mEuM2RuQ9\",\"tx_hosts\":[\"35.199.178.4\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C8I0zn3r9EPbfLgta6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":947,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"79e4a9840d7d3a96d7c04fe2434c892e\",\"sha1\":\"a8985d3a65e5e5c4b2d7d66d40c6dd2fb19c5436\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C8I0zn3r9EPbfLgta6", "category": [ "file" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "server": { @@ -126,17 +130,21 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-04-23T19:56:23.864014900Z", + "ingested": "2021-06-08T07:58:45.531445200Z", + "original": "{\"ts\":1547688801.566262,\"fuid\":\"FShtIS1gydeSFf8M63\",\"tx_hosts\":[\"17.134.127.250\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C6sjVo23iNApLnlAt6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":2089,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"b9742f12eb97eff531d94f7800c6706c\",\"sha1\":\"b88d13fe319d342e7a808ce3a0a1158111fc3c2a\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C6sjVo23iNApLnlAt6", "category": [ "file" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "server": { @@ -195,17 +203,21 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-04-23T19:56:23.864019700Z", + "ingested": "2021-06-08T07:58:45.531448900Z", + "original": "{\"ts\":1547688801.566262,\"fuid\":\"F9ip9a3MDAq3XLBOn2\",\"tx_hosts\":[\"17.134.127.250\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C6sjVo23iNApLnlAt6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":1092,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"48f0e38385112eeca5fc9ffd402eaecd\",\"sha1\":\"8e8321ca08b08e3726fe1d82996884eeb5f0d655\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C6sjVo23iNApLnlAt6", "category": [ "file" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "server": { @@ -268,17 +280,21 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864034100Z", + "ingested": "2021-06-08T07:58:45.531453Z", + "original": "{\"ts\":1617069763.671838,\"fuid\":\"Fe722V1qt2DSlqCiOa\",\"tx_hosts\":[\"104.154.89.105\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"ClG5ErV7bkgKgOaV\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"SHA256\",\"SHA1\",\"MD5\"],\"mime_type\":\"application/x-x509-user-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":893,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"5abbc8a9137f6924861596848b7d7794\",\"sha1\":\"99c1daf07c8d69a8a065492dcaae43c43ff13497\",\"sha256\":\"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "ClG5ErV7bkgKgOaV", "category": [ "file" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "server": { @@ -341,17 +357,21 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864044100Z", + "ingested": "2021-06-08T07:58:45.531457700Z", + "original": "{\"ts\":1617069773.678327,\"fuid\":\"FYszs61e8hIUWMWgL5\",\"tx_hosts\":[\"104.154.89.105\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"CaB3fq3yLrKCbYLqr4\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"SHA256\",\"SHA1\",\"MD5\"],\"mime_type\":\"application/x-x509-user-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":893,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"5abbc8a9137f6924861596848b7d7794\",\"sha1\":\"99c1daf07c8d69a8a065492dcaae43c43ff13497\",\"sha256\":\"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CaB3fq3yLrKCbYLqr4", "category": [ "file" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "server": { @@ -414,17 +434,21 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864048800Z", + "ingested": "2021-06-08T07:58:45.531461700Z", + "original": "{\"ts\":1617069783.678588,\"fuid\":\"FdGWZq2wRIvCfjvdI5\",\"tx_hosts\":[\"104.154.89.105\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"C0vhl91PPOI7LbrPZ8\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"SHA256\",\"SHA1\",\"MD5\"],\"mime_type\":\"application/x-x509-user-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":893,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"5abbc8a9137f6924861596848b7d7794\",\"sha1\":\"99c1daf07c8d69a8a065492dcaae43c43ff13497\",\"sha256\":\"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C0vhl91PPOI7LbrPZ8", "category": [ "file" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "server": { @@ -483,17 +507,21 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864052400Z", + "ingested": "2021-06-08T07:58:45.531468700Z", + "original": "{\"ts\":1617069792.519193,\"fuid\":\"FSMkdM3YUSoEVpLZN4\",\"tx_hosts\":[\"169.254.169.254\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"CgbPEj2jf5Ca7Lw0x2\"],\"source\":\"HTTP\",\"depth\":0,\"analyzers\":[\"SHA1\",\"MD5\"],\"mime_type\":\"text/html\",\"duration\":0.00005316734313964844,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":1609,\"total_bytes\":1609,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"1ab1d3a926a99ccfc25acccc5b4289b4\",\"sha1\":\"1895628784b47ad8da112c699a1b21f5b49c2b80\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CgbPEj2jf5Ca7Lw0x2", "category": [ "file" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "server": { @@ -556,17 +584,21 @@ "ip": "10.156.0.2" }, "event": { - "ingested": "2021-04-23T19:56:23.864055400Z", + "ingested": "2021-06-08T07:58:45.531477500Z", + "original": "{\"ts\":1617069793.669729,\"fuid\":\"F1msmE2xRFsdvL2iI\",\"tx_hosts\":[\"104.154.89.105\"],\"rx_hosts\":[\"10.156.0.2\"],\"conn_uids\":[\"C0vua63rzjtLaiefyj\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"SHA256\",\"SHA1\",\"MD5\"],\"mime_type\":\"application/x-x509-user-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":893,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"5abbc8a9137f6924861596848b7d7794\",\"sha1\":\"99c1daf07c8d69a8a065492dcaae43c43ff13497\",\"sha256\":\"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C0vua63rzjtLaiefyj", "category": [ "file" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "server": { @@ -605,6 +637,9 @@ }, "session_id": "C6sjVo23iNApLnlAt6" }, + "tags": [ + "preserve_original_event" + ], "@timestamp": "2019-01-17T01:33:21.566Z", "file": { "mime_type": "application/pkix-cert", @@ -633,7 +668,7 @@ "ip": "10.178.98.102" }, "event": { - "ingested": "2021-04-23T19:56:23.864058100Z", + "ingested": "2021-06-08T07:58:45.531481600Z", "original": "{\"ts\":1547688801.566262,\"fuid\":\"F9ip9a3MDAq3XLBOn2\",\"tx_hosts\":[\"17.134.127.250\"],\"rx_hosts\":[\"10.178.98.102\"],\"conn_uids\":[\"C6sjVo23iNApLnlAt6\"],\"source\":\"SSL\",\"depth\":0,\"analyzers\":[\"X509\",\"MD5\",\"SHA1\"],\"mime_type\":\"application/pkix-cert\",\"duration\":0.0,\"local_orig\":false,\"is_orig\":false,\"seen_bytes\":1092,\"missing_bytes\":0,\"overflow_bytes\":0,\"timedout\":false,\"md5\":\"48f0e38385112eeca5fc9ffd402eaecd\",\"sha1\":\"8e8321ca08b08e3726fe1d82996884eeb5f0d655\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/files/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/files/agent/stream/log.yml.hbs b/packages/zeek/data_stream/files/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/files/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/files/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml index 8ba137682e5..0d51c3df002 100644 --- a/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/files/elasticsearch/ingest_pipeline/default.yml @@ -20,39 +20,48 @@ processors: - append: field: event.type value: info - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.files - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.files @@ -152,6 +161,11 @@ processors: - json - zeek.files.x509 ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-config.yml b/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-config.yml +++ b/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json b/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json index 269e03042bd..c03f91c138a 100644 --- a/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json +++ b/packages/zeek/data_stream/ftp/_dev/test/pipeline/test-ftp.log-expected.json @@ -1,19 +1,6 @@ { "expected": [ { - "@timestamp": "2007-08-17T19:31:44.955Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "user": [ - "ftp" - ], - "ip": [ - "192.168.1.182", - "192.168.1.231" - ] - }, "destination": { "port": 21, "address": "192.168.1.231", @@ -42,8 +29,30 @@ "address": "192.168.1.182", "ip": "192.168.1.182" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "ftp", + "community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", + "transport": "tcp" + }, + "@timestamp": "2007-08-17T19:31:44.955Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "user": [ + "ftp" + ], + "ip": [ + "192.168.1.182", + "192.168.1.231" + ] + }, "event": { - "ingested": "2021-04-23T19:56:24.111927700Z", + "ingested": "2021-06-08T08:00:52.692863100Z", + "original": "{\"ts\":1187379104.955342,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"EPSV\",\"reply_code\":229,\"reply_msg\":\"Entering Extended Passive Mode (|||37100|)\",\"data_channel.passive\":true,\"data_channel.orig_h\":\"192.168.1.182\",\"data_channel.resp_h\":\"192.168.1.231\",\"data_channel.resp_p\":37100}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "EPSV", @@ -59,11 +68,6 @@ }, "user": { "name": "ftp" - }, - "network": { - "protocol": "ftp", - "community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", - "transport": "tcp" } }, { @@ -90,6 +94,9 @@ "address": "192.168.1.182", "ip": "192.168.1.182" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "ftp", "community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", @@ -112,7 +119,8 @@ ] }, "event": { - "ingested": "2021-04-23T19:56:24.111942800Z", + "ingested": "2021-06-08T08:00:52.692872200Z", + "original": "{\"ts\":1187379105.01948,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"RETR\",\"arg\":\"ftp://192.168.1.231/resume.doc\",\"file_size\":39424,\"reply_code\":226,\"reply_msg\":\"Transfer complete.\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "RETR", @@ -131,19 +139,6 @@ } }, { - "@timestamp": "2007-08-17T19:31:57.579Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "user": [ - "ftp" - ], - "ip": [ - "192.168.1.182", - "192.168.1.231" - ] - }, "destination": { "port": 21, "address": "192.168.1.231", @@ -167,8 +162,30 @@ "address": "192.168.1.182", "ip": "192.168.1.182" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "ftp", + "community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", + "transport": "tcp" + }, + "@timestamp": "2007-08-17T19:31:57.579Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "user": [ + "ftp" + ], + "ip": [ + "192.168.1.182", + "192.168.1.231" + ] + }, "event": { - "ingested": "2021-04-23T19:56:24.111947300Z", + "ingested": "2021-06-08T08:00:52.692879500Z", + "original": "{\"ts\":1187379117.579203,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"STOR\",\"arg\":\"ftp://192.168.1.231/uploads/README\",\"reply_code\":226,\"reply_msg\":\"Transfer complete.\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "STOR", @@ -184,11 +201,6 @@ }, "user": { "name": "ftp" - }, - "network": { - "protocol": "ftp", - "community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", - "transport": "tcp" } }, { @@ -220,6 +232,9 @@ "address": "192.168.1.182", "ip": "192.168.1.182" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "ftp", "community_id": "1:Szmpl33Czo3dQvU2V4/SrHfmBC0=", @@ -242,7 +257,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:24.111950500Z", + "ingested": "2021-06-08T08:00:52.692886500Z", "original": "{\"ts\":1187379117.579203,\"uid\":\"CpQoCn3o28tke89zv9\",\"id.orig_h\":\"192.168.1.182\",\"id.orig_p\":62014,\"id.resp_h\":\"192.168.1.231\",\"id.resp_p\":21,\"user\":\"ftp\",\"password\":\"ftp\",\"command\":\"STOR\",\"arg\":\"ftp://192.168.1.231/uploads/README\",\"reply_code\":226,\"reply_msg\":\"Transfer complete.\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ftp/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/ftp/agent/stream/log.yml.hbs b/packages/zeek/data_stream/ftp/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/ftp/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/ftp/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml index b9334969a5d..ecb1da3d4ca 100644 --- a/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ftp/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: ftp - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.ftp - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.ftp @@ -248,6 +257,11 @@ processors: - json - zeek.ftp.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-config.yml b/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-config.yml +++ b/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json b/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json index 16ca364fe76..e2de238993a 100644 --- a/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json +++ b/packages/zeek/data_stream/http/_dev/test/pipeline/test-http.log-expected.json @@ -49,6 +49,9 @@ "domain": "ocsp.apple.com", "username": "user" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:dtBPRfpKEZyg1iOHss95buwv+cw=", "transport": "tcp" @@ -82,7 +85,8 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.247391800Z", + "ingested": "2021-06-08T08:01:32.385546700Z", + "original": "{\"ts\":1547687130.172944,\"uid\":\"CCNp8v1SNzY7v9d1Ih\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":62995,\"id.resp_h\":\"17.253.5.203\",\"username\":\"user\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"ocsp.apple.com\",\"uri\":\"/ocsp04-aaica02/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFNqvF+Za6oA4ceFRLsAWwEInjUhJBBQx6napI3Sl39T97qDBpp7GEQ4R7AIIUP1IOZZ86ns=\",\"version\":\"1.1\",\"user_agent\":\"com.apple.trustd/2.0\",\"request_body_len\":0,\"response_body_len\":3735,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F5zuip1tSwASjNAHy7\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "get", @@ -157,6 +161,9 @@ "original": "/ip", "domain": "httpbin.org" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:Ol0Btm49e1mxnu/BXm1GM8w5ixY=", "transport": "tcp" @@ -187,7 +194,8 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.247408200Z", + "ingested": "2021-06-08T08:01:32.385556Z", + "original": "{\"ts\":1547707019.757479,\"uid\":\"CMnIaR2V8VXyu7EPs\",\"id.orig_h\":\"10.20.8.197\",\"id.orig_p\":35684,\"id.resp_h\":\"34.206.130.40\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"httpbin.org\",\"uri\":\"/ip\",\"version\":\"1.1\",\"user_agent\":\"curl/7.58.0\",\"request_body_len\":0,\"response_body_len\":32,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FwGPlr1GcKUWWdkXoi\"],\"resp_mime_types\":[\"text/json\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "get", @@ -213,16 +221,6 @@ } }, { - "@timestamp": "2021-03-30T05:15:54.277Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "23.55.163.58" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -257,6 +255,28 @@ }, "session_id": "CdqHhA1AsxBIjmVZ9" }, + "source": { + "port": 57896, + "address": "10.156.0.2", + "ip": "10.156.0.2" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:6EkQLym56b7e/6iC17geVW5hAWc=", + "transport": "tcp" + }, + "@timestamp": "2021-03-30T05:15:54.277Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "23.55.163.58" + ] + }, "http": { "request": { "body": { @@ -271,13 +291,9 @@ "status_code": 200 } }, - "source": { - "port": 57896, - "address": "10.156.0.2", - "ip": "10.156.0.2" - }, "event": { - "ingested": "2021-04-23T19:56:24.247412500Z", + "ingested": "2021-06-08T08:01:32.385559700Z", + "original": "{\"ts\":1617081354.277591,\"uid\":\"CdqHhA1AsxBIjmVZ9\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":57896,\"id.resp_h\":\"23.55.163.58\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.0\",\"request_body_len\":0,\"response_body_len\":503,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FM01o54RU9pez8AJba\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CdqHhA1AsxBIjmVZ9", @@ -291,23 +307,9 @@ "info" ], "outcome": "success" - }, - "network": { - "community_id": "1:6EkQLym56b7e/6iC17geVW5hAWc=", - "transport": "tcp" } }, { - "@timestamp": "2021-03-30T05:15:55.599Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "52.53.69.85" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -345,6 +347,28 @@ }, "session_id": "CxhRTwkHNRsHxBw34" }, + "source": { + "port": 55378, + "address": "10.156.0.2", + "ip": "10.156.0.2" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:DCTMm9iJ3zprWF+EMbM+Kzz2G/g=", + "transport": "tcp" + }, + "@timestamp": "2021-03-30T05:15:55.599Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "52.53.69.85" + ] + }, "http": { "request": { "body": { @@ -359,13 +383,9 @@ "status_code": 301 } }, - "source": { - "port": 55378, - "address": "10.156.0.2", - "ip": "10.156.0.2" - }, "event": { - "ingested": "2021-04-23T19:56:24.247415800Z", + "ingested": "2021-06-08T08:01:32.385564700Z", + "original": "{\"ts\":1617081355.599548,\"uid\":\"CxhRTwkHNRsHxBw34\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":55378,\"id.resp_h\":\"52.53.69.85\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.1\",\"request_body_len\":0,\"response_body_len\":191,\"status_code\":301,\"status_msg\":\"Moved Permanently\",\"tags\":[],\"resp_fuids\":[\"FVGTq31RBgKGE02hx7\"],\"resp_mime_types\":[\"text/html\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CxhRTwkHNRsHxBw34", @@ -379,23 +399,9 @@ "info" ], "outcome": "success" - }, - "network": { - "community_id": "1:DCTMm9iJ3zprWF+EMbM+Kzz2G/g=", - "transport": "tcp" } }, { - "@timestamp": "2021-03-30T05:16:00.171Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "23.55.163.48" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -430,6 +436,28 @@ }, "session_id": "CrI5Xg30caNXnNvEse" }, + "source": { + "port": 41960, + "address": "10.156.0.2", + "ip": "10.156.0.2" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:341n70GKTc+se6UT8lXgjnHVXXo=", + "transport": "tcp" + }, + "@timestamp": "2021-03-30T05:16:00.171Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "23.55.163.48" + ] + }, "http": { "request": { "body": { @@ -444,13 +472,9 @@ "status_code": 200 } }, - "source": { - "port": 41960, - "address": "10.156.0.2", - "ip": "10.156.0.2" - }, "event": { - "ingested": "2021-04-23T19:56:24.247418600Z", + "ingested": "2021-06-08T08:01:32.385569800Z", + "original": "{\"ts\":1617081360.171904,\"uid\":\"CrI5Xg30caNXnNvEse\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":41960,\"id.resp_h\":\"23.55.163.48\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.0\",\"request_body_len\":0,\"response_body_len\":503,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F8vozz46VoxeAmqLv3\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CrI5Xg30caNXnNvEse", @@ -464,23 +488,9 @@ "info" ], "outcome": "success" - }, - "network": { - "community_id": "1:341n70GKTc+se6UT8lXgjnHVXXo=", - "transport": "tcp" } }, { - "@timestamp": "2021-03-30T05:16:04.250Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "23.55.163.48" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -515,6 +525,28 @@ }, "session_id": "C6oCGd24yB2dZ7y7z7" }, + "source": { + "port": 42164, + "address": "10.156.0.2", + "ip": "10.156.0.2" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:82rJ/b6SdSbZMEALyu9kigb2Os0=", + "transport": "tcp" + }, + "@timestamp": "2021-03-30T05:16:04.250Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "23.55.163.48" + ] + }, "http": { "request": { "body": { @@ -529,13 +561,9 @@ "status_code": 200 } }, - "source": { - "port": 42164, - "address": "10.156.0.2", - "ip": "10.156.0.2" - }, "event": { - "ingested": "2021-04-23T19:56:24.247422Z", + "ingested": "2021-06-08T08:01:32.385573200Z", + "original": "{\"ts\":1617081364.250251,\"uid\":\"C6oCGd24yB2dZ7y7z7\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":42164,\"id.resp_h\":\"23.55.163.48\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.0\",\"request_body_len\":0,\"response_body_len\":503,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"F1imAq4yUjbwyK7NO2\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "C6oCGd24yB2dZ7y7z7", @@ -549,23 +577,9 @@ "info" ], "outcome": "success" - }, - "network": { - "community_id": "1:82rJ/b6SdSbZMEALyu9kigb2Os0=", - "transport": "tcp" } }, { - "@timestamp": "2021-03-30T05:16:06.285Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "23.55.163.48" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -600,6 +614,28 @@ }, "session_id": "C7DWRE1zsvxUK9RyW1" }, + "source": { + "port": 42292, + "address": "10.156.0.2", + "ip": "10.156.0.2" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:q4SzvspH9r4RpNUx+pCu9/vYYuQ=", + "transport": "tcp" + }, + "@timestamp": "2021-03-30T05:16:06.285Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "23.55.163.48" + ] + }, "http": { "request": { "body": { @@ -614,13 +650,9 @@ "status_code": 200 } }, - "source": { - "port": 42292, - "address": "10.156.0.2", - "ip": "10.156.0.2" - }, "event": { - "ingested": "2021-04-23T19:56:24.247424800Z", + "ingested": "2021-06-08T08:01:32.385577Z", + "original": "{\"ts\":1617081366.285075,\"uid\":\"C7DWRE1zsvxUK9RyW1\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":42292,\"id.resp_h\":\"23.55.163.48\",\"id.resp_p\":80,\"trans_depth\":1,\"version\":\"1.0\",\"request_body_len\":0,\"response_body_len\":503,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FQhm6z1cISaOxMzzR6\"],\"resp_mime_types\":[\"application/ocsp-response\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "C7DWRE1zsvxUK9RyW1", @@ -634,10 +666,6 @@ "info" ], "outcome": "success" - }, - "network": { - "community_id": "1:q4SzvspH9r4RpNUx+pCu9/vYYuQ=", - "transport": "tcp" } }, { @@ -693,6 +721,9 @@ "original": "/ip", "domain": "httpbin.org" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:Ol0Btm49e1mxnu/BXm1GM8w5ixY=", "transport": "tcp" @@ -726,7 +757,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.247427600Z", + "ingested": "2021-06-08T08:01:32.385580300Z", "original": "{\"ts\":1547707019.757479,\"uid\":\"CMnIaR2V8VXyu7EPs\",\"id.orig_h\":\"10.20.8.197\",\"id.orig_p\":35684,\"id.resp_h\":\"34.206.130.40\",\"id.resp_p\":80,\"trans_depth\":1,\"method\":\"GET\",\"host\":\"httpbin.org\",\"uri\":\"/ip\",\"version\":\"1.1\",\"user_agent\":\"curl/7.58.0\",\"request_body_len\":0,\"response_body_len\":32,\"status_code\":200,\"status_msg\":\"OK\",\"tags\":[],\"resp_fuids\":[\"FwGPlr1GcKUWWdkXoi\"],\"resp_mime_types\":[\"text/json\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/http/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/http/agent/stream/log.yml.hbs b/packages/zeek/data_stream/http/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/http/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/http/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml index 471e756bf69..85b8f986d9e 100644 --- a/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/http/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.transport value: tcp - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.http - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.http @@ -259,6 +268,11 @@ processors: - json - zeek.http.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-config.yml b/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-config.yml +++ b/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json b/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json index 6e04d5df7d4..32c663c8a1e 100644 --- a/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json +++ b/packages/zeek/data_stream/intel/_dev/test/pipeline/test-intel.log-expected.json @@ -54,7 +54,7 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-04-23T19:56:24.549488500Z", + "ingested": "2021-06-03T13:43:29.133730168Z", "original": "{\"ts\":1573030980.989353,\"uid\":\"Ctefoj1tgOPt4D0EK2\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":37598,\"id.resp_h\":\"198.41.0.4\",\"id.resp_p\":53,\"seen.indicator\":\"198.41.0.4\",\"seen.indicator_type\":\"Intel::ADDR\",\"seen.where\":\"Conn::IN_RESP\",\"seen.node\":\"worker-1-2\",\"matched\":[\"Intel::ADDR\"],\"sources\":[\"ETPRO Rep: AbusedTLD Score: 127\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "enrichment", @@ -65,19 +65,12 @@ "type": [ "indicator" ] - } + }, + "tags": [ + "preserve_original_event" + ] }, { - "@timestamp": "2019-11-06T09:03:00.989Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "192.168.1.1", - "198.41.0.4" - ] - }, "log": { "file": { "path": "/usr/local/var/log/zeek/intel.log" @@ -103,9 +96,6 @@ "port": 53, "ip": "198.41.0.4" }, - "host": { - "name": "Lees-MBP.localdomain" - }, "zeek": { "session_id": "Ctefoj1tgOPt4D0EK2", "intel": { @@ -128,8 +118,24 @@ "address": "192.168.1.1", "ip": "192.168.1.1" }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2019-11-06T09:03:00.989Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "192.168.1.1", + "198.41.0.4" + ] + }, + "host": { + "name": "Lees-MBP.localdomain" + }, "event": { - "ingested": "2021-04-23T19:56:24.549496200Z", + "ingested": "2021-06-03T13:43:29.133736987Z", "original": "{\"ts\":1573030980.989353,\"uid\":\"Ctefoj1tgOPt4D0EK2\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":37598,\"id.resp_h\":\"198.41.0.4\",\"id.resp_p\":53,\"seen.indicator\":\"198.41.0.4\",\"seen.indicator_type\":\"Intel::ADDR\",\"seen.where\":\"Conn::IN_RESP\",\"seen.node\":\"worker-1-2\",\"matched\":[\"Intel::ADDR\"],\"sources\":[\"ETPRO Rep: AbusedTLD Score: 127\"]}", "created": "2020-04-28T11:07:58.223Z", "kind": "enrichment", diff --git a/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/intel/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/intel/agent/stream/log.yml.hbs b/packages/zeek/data_stream/intel/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/intel/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/intel/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml index f7512534f3e..e2e656a8800 100644 --- a/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/intel/elasticsearch/ingest_pipeline/default.yml @@ -20,39 +20,48 @@ processors: - append: field: event.type value: indicator - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.intel - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.intel @@ -232,6 +241,11 @@ processors: - json - zeek.intel.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-config.yml b/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-config.yml +++ b/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json b/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json index 6a8860f0574..ec19ed7d3b1 100644 --- a/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json +++ b/packages/zeek/data_stream/irc/_dev/test/pipeline/test-irc.log-expected.json @@ -45,7 +45,8 @@ "ip": "10.180.156.249" }, "event": { - "ingested": "2021-04-23T19:56:24.648599600Z", + "ingested": "2021-06-08T08:02:55.850116500Z", + "original": "{\"ts\":1387554250.647295,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"command\":\"USER\",\"value\":\"xxxxx\",\"addl\":\"+iw xxxxx XxxxxxXxxx \"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "USER", @@ -59,6 +60,9 @@ "info" ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "irc", "community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", @@ -66,19 +70,6 @@ } }, { - "@timestamp": "2013-12-20T15:44:10.647Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "user": [ - "xxxxx" - ], - "ip": [ - "10.180.156.249", - "38.229.70.20" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -112,8 +103,30 @@ "address": "10.180.156.249", "ip": "10.180.156.249" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "irc", + "community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", + "transport": "tcp" + }, + "@timestamp": "2013-12-20T15:44:10.647Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "user": [ + "xxxxx" + ], + "ip": [ + "10.180.156.249", + "38.229.70.20" + ] + }, "event": { - "ingested": "2021-04-23T19:56:24.648608300Z", + "ingested": "2021-06-08T08:02:55.850124600Z", + "original": "{\"ts\":1387554250.647295,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"user\":\"xxxxx\",\"command\":\"NICK\",\"value\":\"molochtest\",\"addl\":\"+iw xxxxx XxxxxxXxxx \"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "NICK", @@ -129,27 +142,9 @@ }, "user": { "name": "xxxxx" - }, - "network": { - "protocol": "irc", - "community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", - "transport": "tcp" } }, { - "@timestamp": "2013-12-20T15:44:10.706Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "user": [ - "xxxxx" - ], - "ip": [ - "10.180.156.249", - "38.229.70.20" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -184,8 +179,30 @@ "address": "10.180.156.249", "ip": "10.180.156.249" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "irc", + "community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", + "transport": "tcp" + }, + "@timestamp": "2013-12-20T15:44:10.706Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "user": [ + "xxxxx" + ], + "ip": [ + "10.180.156.249", + "38.229.70.20" + ] + }, "event": { - "ingested": "2021-04-23T19:56:24.648611Z", + "ingested": "2021-06-08T08:02:55.850132400Z", + "original": "{\"ts\":1387554250.706387,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"nick\":\"molochtest\",\"user\":\"xxxxx\",\"command\":\"JOIN\",\"value\":\"#moloch-fpc\",\"addl\":\" with channel key: \\u0027-\\u0027\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "JOIN", @@ -201,11 +218,6 @@ }, "user": { "name": "xxxxx" - }, - "network": { - "protocol": "irc", - "community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", - "transport": "tcp" } }, { @@ -248,6 +260,9 @@ "address": "10.180.156.249", "ip": "10.180.156.249" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "irc", "community_id": "1:YdkGov/c+KLtmg7Cf5DLDB4+YdQ=", @@ -270,7 +285,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:24.648613100Z", + "ingested": "2021-06-08T08:02:55.850136200Z", "original": "{\"ts\":1387554250.706387,\"uid\":\"CNJBX5FQdL62VUUP1\",\"id.orig_h\":\"10.180.156.249\",\"id.orig_p\":45921,\"id.resp_h\":\"38.229.70.20\",\"id.resp_p\":8000,\"nick\":\"molochtest\",\"user\":\"xxxxx\",\"command\":\"JOIN\",\"value\":\"#moloch-fpc\",\"addl\":\" with channel key: \\u0027-\\u0027\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/irc/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/irc/agent/stream/log.yml.hbs b/packages/zeek/data_stream/irc/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/irc/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/irc/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml index 2c54a34a138..2d4d5b9a26f 100644 --- a/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/irc/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: irc - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.irc - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.irc @@ -205,6 +214,11 @@ processors: - json - zeek.irc.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-config.yml b/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-config.yml +++ b/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json b/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json index 738f8949ae2..605c7642569 100644 --- a/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json +++ b/packages/zeek/data_stream/kerberos/_dev/test/pipeline/test-kerberos.log-expected.json @@ -37,6 +37,9 @@ "address": "192.168.10.31", "ip": "192.168.10.31" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "kerberos", "community_id": "1:DW/lSsosl8gZ8pqO9kKMm7cZheQ=", @@ -83,7 +86,8 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.803165600Z", + "ingested": "2021-06-08T08:03:37.860194Z", + "original": "{\"ts\":1507565599.590346,\"uid\":\"C56Flhb4WQBNkfMOl\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49242,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":88,\"request_type\":\"TGS\",\"client\":\"RonHD/CONTOSO.LOCAL\",\"service\":\"HOST/admin-pc\",\"success\":true,\"till\":2136422885.0,\"cipher\":\"aes256-cts-hmac-sha1-96\",\"forwardable\":true,\"renewable\":true,\"cert.client_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"cert.server_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "TGS", @@ -145,6 +149,9 @@ "address": "192.168.10.31", "ip": "192.168.10.31" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "kerberos", "community_id": "1:DW/lSsosl8gZ8pqO9kKMm7cZheQ=", @@ -194,7 +201,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:24.803185700Z", + "ingested": "2021-06-08T08:03:37.860197700Z", "original": "{\"ts\":1507565599.590346,\"uid\":\"C56Flhb4WQBNkfMOl\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49242,\"id.resp_h\":\"192.168.10.10\",\"id.resp_p\":88,\"request_type\":\"TGS\",\"client\":\"RonHD/CONTOSO.LOCAL\",\"service\":\"HOST/admin-pc\",\"success\":true,\"till\":2136422885.0,\"cipher\":\"aes256-cts-hmac-sha1-96\",\"forwardable\":true,\"renewable\":true,\"cert.client_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"cert.server_subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/kerberos/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/kerberos/agent/stream/log.yml.hbs b/packages/zeek/data_stream/kerberos/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/kerberos/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/kerberos/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml index fcd8c533700..10d51147e0a 100644 --- a/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/kerberos/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: kerberos - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.kerberos - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.kerberos @@ -375,6 +384,11 @@ processors: - json - zeek.kerberos.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-config.yml b/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-config.yml +++ b/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json b/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json index 0053050d03c..96d935f967e 100644 --- a/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json +++ b/packages/zeek/data_stream/modbus/_dev/test/pipeline/test-modbus.log-expected.json @@ -28,7 +28,8 @@ "ip": "192.168.1.10" }, "event": { - "ingested": "2021-04-23T19:56:24.947572900Z", + "ingested": "2021-06-08T08:04:14.373040800Z", + "original": "{\"ts\":1352718265.222457,\"uid\":\"CpIIXl4DFGswmjH2bl\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":64342,\"id.resp_h\":\"192.168.1.164\",\"id.resp_p\":502,\"func\":\"READ_COILS\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "READ_COILS", @@ -42,6 +43,9 @@ ], "outcome": "success" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "modbus", "community_id": "1:jEXbR2FqHyMgLJgyYyFQN3yxbpc=", @@ -70,6 +74,9 @@ "address": "192.168.1.10", "ip": "192.168.1.10" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "modbus", "community_id": "1:jEXbR2FqHyMgLJgyYyFQN3yxbpc=", @@ -89,7 +96,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:24.947580700Z", + "ingested": "2021-06-08T08:04:14.373051100Z", "original": "{\"ts\":1352718265.222457,\"uid\":\"CpIIXl4DFGswmjH2bl\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":64342,\"id.resp_h\":\"192.168.1.164\",\"id.resp_p\":502,\"func\":\"READ_COILS\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/modbus/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/modbus/agent/stream/log.yml.hbs b/packages/zeek/data_stream/modbus/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/modbus/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/modbus/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml index 19e5cc7d035..368116d3b54 100644 --- a/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/modbus/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: modbus - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.modbus - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.modbus @@ -195,6 +204,11 @@ processors: - json - zeek.modbus.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-config.yml b/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-config.yml +++ b/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json b/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json index 583f1ad6530..ba09897ee5e 100644 --- a/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json +++ b/packages/zeek/data_stream/mysql/_dev/test/pipeline/test-mysql.log-expected.json @@ -30,7 +30,8 @@ "ip": "192.168.0.254" }, "event": { - "ingested": "2021-04-23T19:56:25.040662800Z", + "ingested": "2021-06-08T08:04:48.993779100Z", + "original": "{\"ts\":1216281087.437392,\"uid\":\"C5Hol527kLMUw36hj3\",\"id.orig_h\":\"192.168.0.254\",\"id.orig_p\":56162,\"id.resp_h\":\"192.168.0.254\",\"id.resp_p\":3306,\"cmd\":\"query\",\"arg\":\"select count(*) from foo\",\"success\":true,\"rows\":1}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "query", @@ -46,6 +47,9 @@ ], "outcome": "success" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "mysql", "community_id": "1:0HUQbshhYbATQXDHv/ysOs0DlZA=", @@ -77,6 +81,9 @@ "address": "192.168.0.254", "ip": "192.168.0.254" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "mysql", "community_id": "1:0HUQbshhYbATQXDHv/ysOs0DlZA=", @@ -95,7 +102,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.040670800Z", + "ingested": "2021-06-08T08:04:48.993789600Z", "original": "{\"ts\":1216281087.437392,\"uid\":\"C5Hol527kLMUw36hj3\",\"id.orig_h\":\"192.168.0.254\",\"id.orig_p\":56162,\"id.resp_h\":\"192.168.0.254\",\"id.resp_p\":3306,\"cmd\":\"query\",\"arg\":\"select count(*) from foo\",\"success\":true,\"rows\":1}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/mysql/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/mysql/agent/stream/log.yml.hbs b/packages/zeek/data_stream/mysql/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/mysql/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/mysql/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml index 3c0ab2f0d45..3c810cc8bf9 100644 --- a/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/mysql/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: mysql - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.mysql - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.mysql @@ -218,6 +227,11 @@ processors: - json - zeek.mysql.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-config.yml b/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-config.yml +++ b/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json b/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json index 67968376418..3908abb28b0 100644 --- a/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json +++ b/packages/zeek/data_stream/notice/_dev/test/pipeline/test-notice.log-expected.json @@ -32,7 +32,8 @@ "ip": "172.16.238.1" }, "event": { - "ingested": "2021-04-23T19:56:25.173308800Z", + "ingested": "2021-06-08T08:05:32.189738600Z", + "original": "{\"ts\":1320435875.879278,\"note\":\"SSH::Password_Guessing\",\"msg\":\"172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).\",\"sub\":\"Sampled servers: 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136\",\"src\":\"172.16.238.1\",\"peer_descr\":\"bro\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0,\"dropped\":false}", "category": [ "intrusion_detection" ], @@ -42,7 +43,10 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "alert" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2019-02-28T22:36:28.426Z", @@ -117,7 +121,8 @@ "ip": "8.42.77.171" }, "event": { - "ingested": "2021-04-23T19:56:25.173315800Z", + "ingested": "2021-06-08T08:05:32.189748600Z", + "original": "{\"ts\":1551393388.426472,\"note\":\"Scan::Port_Scan\",\"msg\":\"8.42.77.171 scanned at least 15 unique ports of host 207.154.238.205 in 0m0s\",\"sub\":\"remote\",\"src\":\"8.42.77.171\",\"dst\":\"207.154.238.205\",\"peer_descr\":\"bro\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0,\"dropped\":false}", "category": [ "intrusion_detection" ], @@ -127,7 +132,10 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "alert" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2021-03-30T09:49:00.958Z", @@ -149,7 +157,8 @@ "description": "The capture loss script detected an estimated loss rate above 88.306%" }, "event": { - "ingested": "2021-04-23T19:56:25.173320100Z", + "ingested": "2021-06-08T08:05:32.189756Z", + "original": "{\"ts\":1617097740.958466,\"note\":\"CaptureLoss::Too_Much_Loss\",\"msg\":\"The capture loss script detected an estimated loss rate above 88.306%\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0}", "category": [ "intrusion_detection" ], @@ -158,19 +167,12 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "alert" - } + }, + "tags": [ + "preserve_original_event" + ] }, { - "@timestamp": "2021-03-30T09:52:09.601Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "10.156.0.2", - "104.154.89.105" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -216,21 +218,35 @@ "address": "10.156.0.2", "ip": "10.156.0.2" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "community_id": "1:epLQwe8pc8f0Ay9N+VzTshscAGE=", + "transport": "tcp" + }, + "@timestamp": "2021-03-30T09:52:09.601Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "10.156.0.2", + "104.154.89.105" + ] + }, "event": { - "ingested": "2021-04-23T19:56:25.173322600Z", + "ingested": "2021-06-08T08:05:32.189763200Z", + "original": "{\"ts\":1617097929.601155,\"uid\":\"CmvrSS1wIiuOGYCbfi\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":48818,\"id.resp_h\":\"104.154.89.105\",\"id.resp_p\":443,\"fuid\":\"F39b0Bdfm3FW1rNS5\",\"proto\":\"tcp\",\"note\":\"SSL::Invalid_Server_Cert\",\"msg\":\"SSL certificate validation failed with (self signed certificate)\",\"sub\":\"CN=*.badssl.com,O=BadSSL,L=San Francisco,ST=California,C=US\",\"src\":\"10.156.0.2\",\"dst\":\"104.154.89.105\",\"p\":443,\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "alert", "id": "CmvrSS1wIiuOGYCbfi", "category": [ "intrusion_detection" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "alert" - }, - "network": { - "community_id": "1:epLQwe8pc8f0Ay9N+VzTshscAGE=", - "transport": "tcp" + ] } }, { @@ -300,6 +316,9 @@ "address": "8.42.77.171", "ip": "8.42.77.171" }, + "tags": [ + "preserve_original_event" + ], "@timestamp": "2019-02-28T22:36:28.426Z", "ecs": { "version": "1.9.0" @@ -314,7 +333,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.173325600Z", + "ingested": "2021-06-08T08:05:32.189770400Z", "original": "{\"ts\":1551393388.426472,\"note\":\"Scan::Port_Scan\",\"msg\":\"8.42.77.171 scanned at least 15 unique ports of host 207.154.238.205 in 0m0s\",\"sub\":\"remote\",\"src\":\"8.42.77.171\",\"dst\":\"207.154.238.205\",\"peer_descr\":\"bro\",\"actions\":[\"Notice::ACTION_LOG\"],\"suppress_for\":3600.0,\"dropped\":false}", "category": [ "intrusion_detection" diff --git a/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/notice/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/notice/agent/stream/log.yml.hbs b/packages/zeek/data_stream/notice/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/notice/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/notice/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml index dd0d216d3f1..576a53d03ad 100644 --- a/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/notice/elasticsearch/ingest_pipeline/default.yml @@ -20,39 +20,48 @@ processors: - append: field: event.type value: info - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.notice - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.notice @@ -287,6 +296,11 @@ processors: - zeek.notice.remote_location - zeek.notice.f ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-config.yml b/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-config.yml +++ b/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json b/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json index 958ad85f93c..bddb9ce0467 100644 --- a/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json +++ b/packages/zeek/data_stream/ntlm/_dev/test/pipeline/test-ntlm.log-expected.json @@ -1,19 +1,6 @@ { "expected": [ { - "@timestamp": "2017-10-25T19:18:37.814Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "user": [ - "JeffV" - ], - "ip": [ - "192.168.10.50", - "192.168.10.31" - ] - }, "destination": { "port": 445, "address": "192.168.10.31", @@ -39,8 +26,32 @@ "address": "192.168.10.50", "ip": "192.168.10.50" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "ntlm", + "community_id": "1:zxnXAE/Cme5fQhh6sJLs7GItc08=", + "transport": "tcp" + }, + "@timestamp": "2017-10-25T19:18:37.814Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "user": [ + "JeffV" + ], + "ip": [ + "192.168.10.50", + "192.168.10.31" + ] + }, "event": { - "ingested": "2021-04-23T19:56:25.320525100Z", + "ingested": "2021-06-08T08:06:13.747128300Z", + "original": "{\"ts\":1508959117.814467,\"uid\":\"CHphiNUKDC20fsy09\",\"id.orig_h\":\"192.168.10.50\",\"id.orig_p\":46785,\"id.resp_h\":\"192.168.10.31\",\"id.resp_p\":445,\"username\":\"JeffV\",\"hostname\":\"ybaARon55QykXrgu\",\"domainname\":\"contoso.local\",\"server_nb_computer_name\":\"VICTIM-PC\",\"server_dns_computer_name\":\"Victim-PC.contoso.local\",\"server_tree_name\":\"contoso.local\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CHphiNUKDC20fsy09", "category": [ "network", @@ -49,18 +60,11 @@ "type": [ "connection", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, "user": { "name": "JeffV", "domain": "contoso.local" - }, - "network": { - "protocol": "ntlm", - "community_id": "1:zxnXAE/Cme5fQhh6sJLs7GItc08=", - "transport": "tcp" } }, { @@ -94,6 +98,9 @@ "address": "192.168.10.50", "ip": "192.168.10.50" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "ntlm", "community_id": "1:zxnXAE/Cme5fQhh6sJLs7GItc08=", @@ -116,7 +123,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.320530300Z", + "ingested": "2021-06-08T08:06:13.747138600Z", "original": "{\"ts\":1508959117.814467,\"uid\":\"CHphiNUKDC20fsy09\",\"id.orig_h\":\"192.168.10.50\",\"id.orig_p\":46785,\"id.resp_h\":\"192.168.10.31\",\"id.resp_p\":445,\"username\":\"JeffV\",\"hostname\":\"ybaARon55QykXrgu\",\"domainname\":\"contoso.local\",\"server_nb_computer_name\":\"VICTIM-PC\",\"server_dns_computer_name\":\"Victim-PC.contoso.local\",\"server_tree_name\":\"contoso.local\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ntlm/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/ntlm/agent/stream/log.yml.hbs b/packages/zeek/data_stream/ntlm/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/ntlm/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/ntlm/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml index 87b22e3f8ea..56a5835e9fe 100644 --- a/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ntlm/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: ntlm - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.ntlm - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.ntlm @@ -219,6 +228,11 @@ processors: - json - zeek.ntlm.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-config.yml b/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-config.yml +++ b/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json b/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json index c58bc24f95c..3ddf6918e36 100644 --- a/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json +++ b/packages/zeek/data_stream/ocsp/_dev/test/pipeline/test-ocsp.log-expected.json @@ -27,10 +27,14 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.410541800Z", + "ingested": "2021-06-08T08:06:49.721259300Z", + "original": "{\"ts\":1307712421.847886,\"id\":\"FSEWoS3ff8FcTn3WLf\",\"hashAlgorithm\":\"sha1\",\"issuerNameHash\":\"14A7E219F46B93E141258F08BC85764671F136B0\",\"issuerKeyHash\":\"EEDD79C0D379B04D7E47BC70A6E7C62AAEBADEC9\",\"serialNumber\":\"9239D5348F40D1695A745470E1F23F43\",\"certStatus\":\"revoked\",\"revoketime\":1300220120.0,\"thisUpdate\":1307640343.0,\"nextUpdate\":1307985943.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, + "tags": [ + "preserve_original_event" + ], "network": { "transport": "tcp" } @@ -59,10 +63,14 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.410547500Z", + "ingested": "2021-06-08T08:06:49.721268500Z", + "original": "{\"ts\":1307562416.100084,\"id\":\"FdZBFMEYgAErVhoC8\",\"hashAlgorithm\":\"sha1\",\"issuerNameHash\":\"6C2BC55AAF8D96BF60ADF81D023F23B48A0059C2\",\"issuerKeyHash\":\"A5EF0B11CEC04103A34A659048B21CE0572D7D47\",\"serialNumber\":\"30119E6EF41BDBA3FEFE711DBE8F6191\",\"certStatus\":\"good\",\"thisUpdate\":1307549998.0,\"nextUpdate\":1308154798.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, + "tags": [ + "preserve_original_event" + ], "network": { "transport": "tcp" } @@ -99,11 +107,14 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.410549800Z", + "ingested": "2021-06-08T08:06:49.721275400Z", "original": "{\"ts\":1307562416.100084,\"id\":\"FdZBFMEYgAErVhoC8\",\"hashAlgorithm\":\"sha1\",\"issuerNameHash\":\"6C2BC55AAF8D96BF60ADF81D023F23B48A0059C2\",\"issuerKeyHash\":\"A5EF0B11CEC04103A34A659048B21CE0572D7D47\",\"serialNumber\":\"30119E6EF41BDBA3FEFE711DBE8F6191\",\"certStatus\":\"good\",\"thisUpdate\":1307549998.0,\"nextUpdate\":1308154798.0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, + "tags": [ + "preserve_original_event" + ], "network": { "transport": "tcp" } diff --git a/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ocsp/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/ocsp/agent/stream/log.yml.hbs b/packages/zeek/data_stream/ocsp/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/ocsp/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/ocsp/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml index be7be34575c..deef718d6e0 100644 --- a/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ocsp/elasticsearch/ingest_pipeline/default.yml @@ -17,39 +17,48 @@ processors: - set: field: network.transport value: tcp - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.ocsp - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.ocsp @@ -134,6 +143,11 @@ processors: - message - json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-config.yml b/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-config.yml +++ b/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json b/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json index b89ce628d27..c931b88e9ee 100644 --- a/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json +++ b/packages/zeek/data_stream/pe/_dev/test/pipeline/test-pe.log-expected.json @@ -32,7 +32,8 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.485194600Z", + "ingested": "2021-06-08T08:07:29.500985800Z", + "original": "{\"ts\":1507565599.578328,\"id\":\"FtIFnm3ZqI1s96P74l\",\"machine\":\"I386\",\"compile_ts\":1467139314.0,\"os\":\"Windows XP\",\"subsystem\":\"WINDOWS_CUI\",\"is_exe\":true,\"is_64bit\":false,\"uses_aslr\":true,\"uses_dep\":true,\"uses_code_integrity\":false,\"uses_seh\":true,\"has_import_table\":true,\"has_export_table\":false,\"has_cert_table\":true,\"has_debug_data\":false,\"section_names\":[\".text\",\".rdata\",\".data\",\".rsrc\",\".reloc\"]}", "category": [ "file" ], @@ -41,7 +42,10 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2017-10-09T16:13:19.578Z", @@ -83,7 +87,7 @@ } }, "event": { - "ingested": "2021-04-23T19:56:25.485200900Z", + "ingested": "2021-06-08T08:07:29.501025900Z", "original": "{\"ts\":1507565599.578328,\"id\":\"FtIFnm3ZqI1s96P74l\",\"machine\":\"I386\",\"compile_ts\":1467139314.0,\"os\":\"Windows XP\",\"subsystem\":\"WINDOWS_CUI\",\"is_exe\":true,\"is_64bit\":false,\"uses_aslr\":true,\"uses_dep\":true,\"uses_code_integrity\":false,\"uses_seh\":true,\"has_import_table\":true,\"has_export_table\":false,\"has_cert_table\":true,\"has_debug_data\":false,\"section_names\":[\".text\",\".rdata\",\".data\",\".rsrc\",\".reloc\"]}", "category": [ "file" @@ -93,7 +97,10 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/pe/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/pe/agent/stream/log.yml.hbs b/packages/zeek/data_stream/pe/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/pe/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/pe/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml index ce8d0119aa4..2426bf9da56 100644 --- a/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/pe/elasticsearch/ingest_pipeline/default.yml @@ -20,39 +20,48 @@ processors: - append: field: event.type value: info - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.pe - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.pe @@ -79,6 +88,11 @@ processors: - message - json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-config.yml b/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-config.yml +++ b/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json b/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json index c8fe6b9ff82..63434e9b709 100644 --- a/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json +++ b/packages/zeek/data_stream/radius/_dev/test/pipeline/test-radius.log-expected.json @@ -1,19 +1,6 @@ { "expected": [ { - "@timestamp": "2008-08-01T22:52:17.916Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "user": [ - "John.McGuirk" - ], - "ip": [ - "10.0.0.1", - "10.0.0.100" - ] - }, "destination": { "port": 1812, "address": "10.0.0.100", @@ -32,8 +19,30 @@ "address": "10.0.0.1", "ip": "10.0.0.1" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "radius", + "community_id": "1:3SdDgWXPnheV2oGfVmxQjfwtr8E=", + "transport": "udp" + }, + "@timestamp": "2008-08-01T22:52:17.916Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "user": [ + "John.McGuirk" + ], + "ip": [ + "10.0.0.1", + "10.0.0.100" + ] + }, "event": { - "ingested": "2021-04-23T19:56:25.555260Z", + "ingested": "2021-06-08T08:20:13.508781400Z", + "original": "{\"ts\":1217631137.916736,\"uid\":\"CRe9VD3flCDWbPmpIh\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":1645,\"id.resp_h\":\"10.0.0.100\",\"id.resp_p\":1812,\"username\":\"John.McGuirk\",\"mac\":\"00:14:22:e9:54:5e\",\"result\":\"success\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CRe9VD3flCDWbPmpIh", @@ -49,11 +58,6 @@ }, "user": { "name": "John.McGuirk" - }, - "network": { - "protocol": "radius", - "community_id": "1:3SdDgWXPnheV2oGfVmxQjfwtr8E=", - "transport": "udp" } }, { @@ -80,6 +84,9 @@ "address": "10.0.0.1", "ip": "10.0.0.1" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "radius", "community_id": "1:3SdDgWXPnheV2oGfVmxQjfwtr8E=", @@ -102,7 +109,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.555267700Z", + "ingested": "2021-06-08T08:20:13.508792200Z", "original": "{\"ts\":1217631137.916736,\"uid\":\"CRe9VD3flCDWbPmpIh\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":1645,\"id.resp_h\":\"10.0.0.100\",\"id.resp_p\":1812,\"username\":\"John.McGuirk\",\"mac\":\"00:14:22:e9:54:5e\",\"result\":\"success\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/radius/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/radius/agent/stream/log.yml.hbs b/packages/zeek/data_stream/radius/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/radius/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/radius/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml index f46ef06438f..6b04a27ca34 100644 --- a/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/radius/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: radius - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.radius - if: 'ctx?.json?.ts != null' + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.radius @@ -195,6 +204,11 @@ processors: - json - zeek.radius.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-config.yml b/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-config.yml +++ b/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json b/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json index 86ba69d1d91..a3b3ca55778 100644 --- a/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json +++ b/packages/zeek/data_stream/rdp/_dev/test/pipeline/test-rdp.log-expected.json @@ -1,16 +1,6 @@ { "expected": [ { - "@timestamp": "2019-09-10T16:18:59.668Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "192.168.131.1", - "192.168.131.131" - ] - }, "destination": { "port": 3389, "address": "192.168.131.131", @@ -27,16 +17,37 @@ "ssl": true } }, - "tls": { - "established": true - }, "source": { "port": 33872, "address": "192.168.131.1", "ip": "192.168.131.1" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "rdp", + "community_id": "1:PsQu6lSZioPVi0A5K7UaeGsVqS0=", + "transport": "tcp" + }, + "@timestamp": "2019-09-10T16:18:59.668Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "192.168.131.1", + "192.168.131.131" + ] + }, + "tls": { + "established": true + }, "event": { - "ingested": "2021-04-23T19:56:25.633749700Z", + "ingested": "2021-06-08T08:24:01.785311100Z", + "original": "{\"ts\":1568132339.668952,\"uid\":\"C2PcYV7D3ntaHm056\",\"id.orig_h\":\"192.168.131.1\",\"id.orig_p\":33872,\"id.resp_h\":\"192.168.131.131\",\"id.resp_p\":3389,\"result\":\"encrypted\",\"security_protocol\":\"HYBRID\",\"cert_count\":0,\"ssl\":true}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C2PcYV7D3ntaHm056", "category": [ "network" @@ -44,14 +55,7 @@ "type": [ "protocol", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - }, - "network": { - "protocol": "rdp", - "community_id": "1:PsQu6lSZioPVi0A5K7UaeGsVqS0=", - "transport": "tcp" + ] } }, { @@ -81,6 +85,9 @@ "address": "192.168.131.1", "ip": "192.168.131.1" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "rdp", "community_id": "1:PsQu6lSZioPVi0A5K7UaeGsVqS0=", @@ -103,7 +110,7 @@ "established": true }, "event": { - "ingested": "2021-04-23T19:56:25.633754800Z", + "ingested": "2021-06-08T08:24:01.785319500Z", "original": "{\"ts\":1568132339.668952,\"uid\":\"C2PcYV7D3ntaHm056\",\"id.orig_h\":\"192.168.131.1\",\"id.orig_p\":33872,\"id.resp_h\":\"192.168.131.131\",\"id.resp_p\":3389,\"result\":\"encrypted\",\"security_protocol\":\"HYBRID\",\"cert_count\":0,\"ssl\":true}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..9bead572996 100644 --- a/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/rdp/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} \ No newline at end of file diff --git a/packages/zeek/data_stream/rdp/agent/stream/log.yml.hbs b/packages/zeek/data_stream/rdp/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/rdp/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/rdp/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml index fc9f9bd8a53..a7f6a107d17 100644 --- a/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/rdp/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: rdp - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.rdp - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.rdp @@ -228,6 +237,11 @@ processors: - json - zeek.rdp.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-config.yml b/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-config.yml +++ b/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json b/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json index 1d341e31c2b..f6fb5938020 100644 --- a/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json +++ b/packages/zeek/data_stream/rfb/_dev/test/pipeline/test-rfb.log-expected.json @@ -45,7 +45,10 @@ "ip": "192.168.1.123" }, "event": { - "ingested": "2021-04-23T19:56:25.723384600Z", + "ingested": "2021-06-08T08:24:28.464611100Z", + "original": "{\"ts\":1328632534.517208,\"uid\":\"CXoIzM3wH3fUwXtKN1\",\"id.orig_h\":\"192.168.1.123\",\"id.orig_p\":58102,\"id.resp_h\":\"192.168.1.10\",\"id.resp_p\":5900,\"client_major_version\":\"003\",\"client_minor_version\":\"008\",\"server_major_version\":\"003\",\"server_minor_version\":\"008\",\"authentication_method\":\"VNC\",\"auth\":true,\"share_flag\":false,\"desktop_name\":\"\\u00a0\",\"width\":800,\"height\":600}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CXoIzM3wH3fUwXtKN1", "category": [ "network" @@ -53,10 +56,11 @@ "type": [ "connection", "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "rfb", "community_id": "1:AtPVA5phuztnwqMfO/2142WXVdY=", @@ -102,6 +106,9 @@ "address": "192.168.1.123", "ip": "192.168.1.123" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "rfb", "community_id": "1:AtPVA5phuztnwqMfO/2142WXVdY=", @@ -121,7 +128,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.723391300Z", + "ingested": "2021-06-08T08:24:28.464620100Z", "original": "{\"ts\":1328632534.517208,\"uid\":\"CXoIzM3wH3fUwXtKN1\",\"id.orig_h\":\"192.168.1.123\",\"id.orig_p\":58102,\"id.resp_h\":\"192.168.1.10\",\"id.resp_p\":5900,\"client_major_version\":\"003\",\"client_minor_version\":\"008\",\"server_major_version\":\"003\",\"server_minor_version\":\"008\",\"authentication_method\":\"VNC\",\"auth\":true,\"share_flag\":false,\"desktop_name\":\"\\u00a0\",\"width\":800,\"height\":600}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/rfb/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/rfb/agent/stream/log.yml.hbs b/packages/zeek/data_stream/rfb/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/rfb/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/rfb/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml index 554ccb775b0..aa5d7e958a8 100644 --- a/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/rfb/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: rfb - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.rfb - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.rfb @@ -203,6 +212,11 @@ processors: - json - zeek.rfb.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-config.yml b/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-config.yml +++ b/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json b/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json index b4379b28720..2f229584df9 100644 --- a/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json +++ b/packages/zeek/data_stream/sip/_dev/test/pipeline/test-sip.log-expected.json @@ -1,16 +1,6 @@ { "expected": [ { - "@timestamp": "2013-02-26T22:02:39.055Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "172.16.133.19", - "74.63.41.218" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -69,8 +59,30 @@ "address": "172.16.133.19", "ip": "172.16.133.19" }, + "url": { + "full": "sip:newyork.voip.ms:5060" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "sip", + "community_id": "1:t8Jl0amIXPHemzxKgsLjtkB+ewo=", + "transport": "udp" + }, + "@timestamp": "2013-02-26T22:02:39.055Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "172.16.133.19", + "74.63.41.218" + ] + }, "event": { - "ingested": "2021-04-23T19:56:25.834066Z", + "ingested": "2021-06-08T08:24:55.665568100Z", + "original": "{\"ts\":1361916159.055464,\"uid\":\"CPRLCB4eWHdjP852Bk\",\"id.orig_h\":\"172.16.133.19\",\"id.orig_p\":5060,\"id.resp_h\":\"74.63.41.218\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"REGISTER\",\"uri\":\"sip:newyork.voip.ms:5060\",\"request_from\":\"\\u0022AppNeta\\u0022 \u003csip:116954_Boston6@newyork.voip.ms\u003e\",\"request_to\":\"\u003csip:116954_Boston6@newyork.voip.ms\u003e\",\"response_from\":\"\\u0022AppNeta\\u0022 \u003csip:116954_Boston6@newyork.voip.ms\u003e\",\"response_to\":\"\u003csip:116954_Boston6@newyork.voip.ms\u003e;tag=as023f66a5\",\"call_id\":\"8694cd7e-976e4fc3-d76f6e38@172.16.133.19\",\"seq\":\"4127 REGISTER\",\"request_path\":[\"SIP/2.0/UDP 172.16.133.19:5060\"],\"response_path\":[\"SIP/2.0/UDP 172.16.133.19:5060\"],\"user_agent\":\"PolycomSoundStationIP-SSIP_5000-UA/3.2.4.0267\",\"status_code\":401,\"status_msg\":\"Unauthorized\",\"request_body_len\":0,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "REGISTER", @@ -84,27 +96,9 @@ "error" ], "outcome": "failure" - }, - "url": { - "full": "sip:newyork.voip.ms:5060" - }, - "network": { - "protocol": "sip", - "community_id": "1:t8Jl0amIXPHemzxKgsLjtkB+ewo=", - "transport": "udp" } }, { - "@timestamp": "2005-01-14T17:58:02.965Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "200.57.7.204", - "200.57.7.195" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -187,8 +181,30 @@ "port": 5061, "ip": "200.57.7.204" }, + "url": { + "full": "sip:francisco@bestel.com:55060" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "sip", + "community_id": "1:U/Makwsc8lm6pVKLfRMzoNTI++0=", + "transport": "udp" + }, + "@timestamp": "2005-01-14T17:58:02.965Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "200.57.7.204", + "200.57.7.195" + ] + }, "event": { - "ingested": "2021-04-23T19:56:25.834072100Z", + "ingested": "2021-06-08T08:24:55.665577100Z", + "original": "{\"ts\":1105725482.965944,\"uid\":\"ComJz236lSOcuOmix3\",\"id.orig_h\":\"200.57.7.204\",\"id.orig_p\":5061,\"id.resp_h\":\"200.57.7.195\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"INVITE\",\"uri\":\"sip:francisco@bestel.com:55060\",\"request_from\":\"\u003csip:200.57.7.195:55061;user=phone\u003e\",\"request_to\":\"\\u0022francisco@bestel.com\\u0022 \u003csip:francisco@bestel.com:55060\u003e\",\"response_from\":\"\u003csip:200.57.7.195:55061;user=phone\u003e\",\"response_to\":\"\\u0022francisco@bestel.com\\u0022 \u003csip:francisco@bestel.com:55060\u003e;tag=298852044\",\"call_id\":\"12013223@200.57.7.195\",\"seq\":\"1 INVITE\",\"request_path\":[\"SIP/2.0/UDP 200.57.7.195\",\"SIP/2.0/UDP 200.57.7.195:55061\"],\"response_path\":[\"SIP/2.0/UDP 200.57.7.195\",\"SIP/2.0/UDP 200.57.7.195:55061\",\"SIP/2.0/UDP 200.57.7.195\",\"SIP/2.0/UDP 200.57.7.195:55061\"],\"status_code\":180,\"status_msg\":\"Ringing\",\"request_body_len\":229,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "INVITE", @@ -201,27 +217,9 @@ "protocol" ], "outcome": "success" - }, - "url": { - "full": "sip:francisco@bestel.com:55060" - }, - "network": { - "protocol": "sip", - "community_id": "1:U/Makwsc8lm6pVKLfRMzoNTI++0=", - "transport": "udp" } }, { - "@timestamp": "2005-01-14T17:58:07.022Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "200.57.7.205", - "200.57.7.195" - ] - }, "destination": { "geo": { "continent_name": "North America", @@ -301,8 +299,30 @@ "port": 5061, "ip": "200.57.7.205" }, + "url": { + "full": "sip:Verso.com" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "sip", + "community_id": "1:0hvHF/bh5wFKg7nfRXxsno4F198=", + "transport": "udp" + }, + "@timestamp": "2005-01-14T17:58:07.022Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "200.57.7.205", + "200.57.7.195" + ] + }, "event": { - "ingested": "2021-04-23T19:56:25.834128800Z", + "ingested": "2021-06-08T08:24:55.665584100Z", + "original": "{\"ts\":1105725487.022577,\"uid\":\"CJZDWgixtwqXctWEg\",\"id.orig_h\":\"200.57.7.205\",\"id.orig_p\":5061,\"id.resp_h\":\"200.57.7.195\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"REGISTER\",\"uri\":\"sip:Verso.com\",\"request_from\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"request_to\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"response_from\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"response_to\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"call_id\":\"46E1C3CB36304F84A020CF6DD3F96461@Verso.com\",\"seq\":\"37764 REGISTER\",\"request_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;rport\"],\"response_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;received=200.57.7.205;rport=5061\"],\"user_agent\":\"Verso Softphone release 1104w\",\"status_code\":200,\"status_msg\":\"OK\",\"request_body_len\":0,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "REGISTER", @@ -315,27 +335,9 @@ "protocol" ], "outcome": "success" - }, - "url": { - "full": "sip:Verso.com" - }, - "network": { - "protocol": "sip", - "community_id": "1:0hvHF/bh5wFKg7nfRXxsno4F198=", - "transport": "udp" } }, { - "@timestamp": "2021-03-30T15:50:16.928Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "193.107.216.13", - "10.156.0.2" - ] - }, "destination": { "port": 5060, "address": "10.156.0.2", @@ -385,40 +387,44 @@ "port": 5083, "ip": "193.107.216.13" }, - "event": { - "ingested": "2021-04-23T19:56:25.834134900Z", - "created": "2020-04-28T11:07:58.223Z", - "kind": "event", - "action": "OPTIONS", - "id": "CR6XQH1Lf2mF9YG7H2", - "category": [ - "network" - ], - "type": [ - "connection", - "protocol" - ] - }, "url": { "full": "sip:100@35.198.74.222" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "sip", "community_id": "1:0yHuzsMc9NWnZAgB15XTv5hKFPI=", "transport": "udp" - } - }, - { - "@timestamp": "2021-03-30T15:58:43.416Z", + }, + "@timestamp": "2021-03-30T15:50:16.928Z", "ecs": { "version": "1.9.0" }, "related": { "ip": [ - "45.134.144.100", + "193.107.216.13", "10.156.0.2" ] }, + "event": { + "ingested": "2021-06-08T08:24:55.665603900Z", + "original": "{\"ts\":1617119416.928735,\"uid\":\"CR6XQH1Lf2mF9YG7H2\",\"id.orig_h\":\"193.107.216.13\",\"id.orig_p\":5083,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"OPTIONS\",\"uri\":\"sip:100@35.198.74.222\",\"request_from\":\"\\\"sipvicious\\\"\u003csip:90501@1.1.1.1\u003e\",\"request_to\":\"\\\"sipvicious\\\"\u003csip:90501@1.1.1.1\u003e\",\"call_id\":\"767538559354206383610151\",\"seq\":\"1 OPTIONS\",\"request_path\":[\"SIP/2.0/UDP 193.107.216.13:5083\"],\"response_path\":[],\"user_agent\":\"friendly-scanner\",\"request_body_len\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "action": "OPTIONS", + "id": "CR6XQH1Lf2mF9YG7H2", + "category": [ + "network" + ], + "type": [ + "connection", + "protocol" + ] + } + }, + { "destination": { "port": 5060, "address": "10.156.0.2", @@ -468,8 +474,30 @@ "port": 5170, "ip": "45.134.144.100" }, + "url": { + "full": "sip:100@35.198.74.222" + }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "sip", + "community_id": "1:CG92d5aAL3DgFhEJiDndd41USVA=", + "transport": "udp" + }, + "@timestamp": "2021-03-30T15:58:43.416Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "45.134.144.100", + "10.156.0.2" + ] + }, "event": { - "ingested": "2021-04-23T19:56:25.834137600Z", + "ingested": "2021-06-08T08:24:55.665608900Z", + "original": "{\"ts\":1617119923.416653,\"uid\":\"Cf9QMt4ear7ZkX74ti\",\"id.orig_h\":\"45.134.144.100\",\"id.orig_p\":5170,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"OPTIONS\",\"uri\":\"sip:100@35.198.74.222\",\"request_from\":\"\\\"sipvicious\\\"\u003csip:100@1.1.1.1\u003e\",\"request_to\":\"\\\"sipvicious\\\"\u003csip:100@1.1.1.1\u003e\",\"call_id\":\"35848812076538877174452\",\"seq\":\"1 OPTIONS\",\"request_path\":[\"SIP/2.0/UDP 127.0.0.1:5170\"],\"response_path\":[],\"user_agent\":\"friendly-scanner\",\"request_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "OPTIONS", @@ -481,14 +509,6 @@ "connection", "protocol" ] - }, - "url": { - "full": "sip:100@35.198.74.222" - }, - "network": { - "protocol": "sip", - "community_id": "1:CG92d5aAL3DgFhEJiDndd41USVA=", - "transport": "udp" } }, { @@ -579,6 +599,9 @@ "url": { "full": "sip:Verso.com" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "sip", "community_id": "1:0hvHF/bh5wFKg7nfRXxsno4F198=", @@ -598,7 +621,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:25.834139700Z", + "ingested": "2021-06-08T08:24:55.665613700Z", "original": "{\"ts\":1105725487.022577,\"uid\":\"CJZDWgixtwqXctWEg\",\"id.orig_h\":\"200.57.7.205\",\"id.orig_p\":5061,\"id.resp_h\":\"200.57.7.195\",\"id.resp_p\":5060,\"trans_depth\":0,\"method\":\"REGISTER\",\"uri\":\"sip:Verso.com\",\"request_from\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"request_to\":\"Ivan \u003csip:Ivan@Verso.com\u003e\",\"response_from\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"response_to\":\"\\u0022Ivan\\u0022 \u003csip:Ivan@Verso.com\u003e\",\"call_id\":\"46E1C3CB36304F84A020CF6DD3F96461@Verso.com\",\"seq\":\"37764 REGISTER\",\"request_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;rport\"],\"response_path\":[\"SIP/2.0/UDP 200.57.7.205:5061;received=200.57.7.205;rport=5061\"],\"user_agent\":\"Verso Softphone release 1104w\",\"status_code\":200,\"status_msg\":\"OK\",\"request_body_len\":0,\"response_body_len\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/sip/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/sip/agent/stream/log.yml.hbs b/packages/zeek/data_stream/sip/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/sip/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/sip/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml index 3d307c2b4f9..c28e2b92703 100644 --- a/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/sip/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: sip - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.sip - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.sip @@ -255,6 +264,11 @@ processors: - json - zeek.sip.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-config.yml b/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-config.yml +++ b/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json b/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json index fe6fbca61e5..08cf9e8b9d8 100644 --- a/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json +++ b/packages/zeek/data_stream/smb_cmd/_dev/test/pipeline/test-smb-cmd.log-expected.json @@ -43,7 +43,8 @@ "ip": "172.16.133.6" }, "event": { - "ingested": "2021-04-23T19:56:26.060942200Z", + "ingested": "2021-06-08T08:25:28.423288800Z", + "original": "{\"ts\":1361916332.020006,\"uid\":\"CbT8mpAXseu6Pt4R7\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"command\":\"NT_CREATE_ANDX\",\"argument\":\"\\u005cbrowser\",\"status\":\"SUCCESS\",\"rtt\":0.091141,\"version\":\"SMB1\",\"tree\":\"\\u005c\\u005cJSRVR20\\u005cIPC$\",\"tree_service\":\"IPC\",\"referenced_file.ts\":1361916332.020006,\"referenced_file.uid\":\"CbT8mpAXseu6Pt4R7\",\"referenced_file.id.orig_h\":\"172.16.133.6\",\"referenced_file.id.orig_p\":1728,\"referenced_file.id.resp_h\":\"172.16.128.202\",\"referenced_file.id.resp_p\":445,\"referenced_file.action\":\"SMB::FILE_OPEN\",\"referenced_file.name\":\"\\u005cbrowser\",\"referenced_file.size\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "NT_CREATE_ANDX", @@ -57,6 +58,9 @@ ], "outcome": "success" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "smb", "community_id": "1:SJNAD5vtzZuhQjGtfaI8svTnyuw=", @@ -100,6 +104,9 @@ "address": "172.16.133.6", "ip": "172.16.133.6" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "smb", "community_id": "1:SJNAD5vtzZuhQjGtfaI8svTnyuw=", @@ -119,7 +126,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.060951300Z", + "ingested": "2021-06-08T08:25:28.423299Z", "original": "{\"ts\":1361916332.020006,\"uid\":\"CbT8mpAXseu6Pt4R7\",\"id.orig_h\":\"172.16.133.6\",\"id.orig_p\":1728,\"id.resp_h\":\"172.16.128.202\",\"id.resp_p\":445,\"command\":\"NT_CREATE_ANDX\",\"argument\":\"\\u005cbrowser\",\"status\":\"SUCCESS\",\"rtt\":0.091141,\"version\":\"SMB1\",\"tree\":\"\\u005c\\u005cJSRVR20\\u005cIPC$\",\"tree_service\":\"IPC\",\"referenced_file.ts\":1361916332.020006,\"referenced_file.uid\":\"CbT8mpAXseu6Pt4R7\",\"referenced_file.id.orig_h\":\"172.16.133.6\",\"referenced_file.id.orig_p\":1728,\"referenced_file.id.resp_h\":\"172.16.128.202\",\"referenced_file.id.resp_p\":445,\"referenced_file.action\":\"SMB::FILE_OPEN\",\"referenced_file.name\":\"\\u005cbrowser\",\"referenced_file.size\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/smb_cmd/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/smb_cmd/agent/stream/log.yml.hbs b/packages/zeek/data_stream/smb_cmd/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/smb_cmd/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/smb_cmd/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml index 3c726db949a..0603b352f99 100644 --- a/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_cmd/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: smb - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.smb_cmd - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.smb_cmd @@ -305,6 +314,11 @@ processors: - json - zeek.smb_cmd.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-config.yml b/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-config.yml +++ b/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json b/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json index 69ee6c24bb0..f015498d907 100644 --- a/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json +++ b/packages/zeek/data_stream/smb_files/_dev/test/pipeline/test-smb-files.log-expected.json @@ -1,25 +1,6 @@ { "expected": [ { - "@timestamp": "2017-10-09T16:13:19.576Z", - "file": { - "path": "\\\\\\\\admin-pc\\\\ADMIN$\\PSEXESVC.exe", - "size": 0, - "created": "2017-10-09T16:13:19.607Z", - "name": "PSEXESVC.exe", - "ctime": "2017-10-09T16:13:19.607Z", - "accessed": "2017-10-09T16:13:19.607Z", - "mtime": "2017-10-09T16:13:19.607Z" - }, - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "192.168.10.31", - "192.168.10.30" - ] - }, "destination": { "port": 445, "address": "192.168.10.30", @@ -45,8 +26,36 @@ "address": "192.168.10.31", "ip": "192.168.10.31" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "smb", + "community_id": "1:k308wDxRMx/FIEzeh+YwD86zgoA=", + "transport": "tcp" + }, + "@timestamp": "2017-10-09T16:13:19.576Z", + "file": { + "path": "\\\\\\\\admin-pc\\\\ADMIN$\\PSEXESVC.exe", + "size": 0, + "created": "2017-10-09T16:13:19.607Z", + "name": "PSEXESVC.exe", + "ctime": "2017-10-09T16:13:19.607Z", + "accessed": "2017-10-09T16:13:19.607Z", + "mtime": "2017-10-09T16:13:19.607Z" + }, + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "192.168.10.31", + "192.168.10.30" + ] + }, "event": { - "ingested": "2021-04-23T19:56:26.168856300Z", + "ingested": "2021-06-08T08:25:55.006846100Z", + "original": "{\"ts\":1507565599.576942,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"action\":\"SMB::FILE_OPEN\",\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"name\":\"PSEXESVC.exe\",\"size\":0,\"times.modified\":1507565599.607777,\"times.accessed\":1507565599.607777,\"times.created\":1507565599.607777,\"times.changed\":1507565599.607777}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "action": "SMB::FILE_OPEN", @@ -60,11 +69,6 @@ "protocol", "info" ] - }, - "network": { - "protocol": "smb", - "community_id": "1:k308wDxRMx/FIEzeh+YwD86zgoA=", - "transport": "tcp" } }, { @@ -98,6 +102,9 @@ "address": "192.168.10.31", "ip": "192.168.10.31" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "smb", "community_id": "1:k308wDxRMx/FIEzeh+YwD86zgoA=", @@ -126,7 +133,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.168863Z", + "ingested": "2021-06-08T08:25:55.006850600Z", "original": "{\"ts\":1507565599.576942,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"action\":\"SMB::FILE_OPEN\",\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"name\":\"PSEXESVC.exe\",\"size\":0,\"times.modified\":1507565599.607777,\"times.accessed\":1507565599.607777,\"times.created\":1507565599.607777,\"times.changed\":1507565599.607777}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/smb_files/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/smb_files/agent/stream/log.yml.hbs b/packages/zeek/data_stream/smb_files/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/smb_files/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/smb_files/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml index f986f2cdea5..09a524f5a61 100644 --- a/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_files/elasticsearch/ingest_pipeline/default.yml @@ -32,39 +32,48 @@ processors: - set: field: network.protocol value: smb - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.smb_files - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.smb_files @@ -267,6 +276,11 @@ processors: - json - zeek.smb_files.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-config.yml b/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-config.yml +++ b/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json b/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json index 9a899e272d4..92fdf09a735 100644 --- a/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json +++ b/packages/zeek/data_stream/smb_mapping/_dev/test/pipeline/test-smb-mapping.log-expected.json @@ -29,7 +29,10 @@ "ip": "192.168.10.31" }, "event": { - "ingested": "2021-04-23T19:56:26.260175700Z", + "ingested": "2021-06-08T08:26:26.606990700Z", + "original": "{\"ts\":1507565599.576613,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"share_type\":\"DISK\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C9YAaEzWLL62yWMn5", "category": [ "network" @@ -37,10 +40,11 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "smb", "community_id": "1:k308wDxRMx/FIEzeh+YwD86zgoA=", @@ -70,6 +74,9 @@ "address": "192.168.10.31", "ip": "192.168.10.31" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "smb", "community_id": "1:k308wDxRMx/FIEzeh+YwD86zgoA=", @@ -89,7 +96,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.260180800Z", + "ingested": "2021-06-08T08:26:26.606996Z", "original": "{\"ts\":1507565599.576613,\"uid\":\"C9YAaEzWLL62yWMn5\",\"id.orig_h\":\"192.168.10.31\",\"id.orig_p\":49239,\"id.resp_h\":\"192.168.10.30\",\"id.resp_p\":445,\"path\":\"\\u005c\\u005cadmin-pc\\u005cADMIN$\",\"share_type\":\"DISK\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/smb_mapping/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/smb_mapping/agent/stream/log.yml.hbs b/packages/zeek/data_stream/smb_mapping/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/smb_mapping/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/smb_mapping/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml index 43cbc0d729b..f00762ad0b9 100644 --- a/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smb_mapping/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: smb - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.smb_mapping - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.smb_mapping @@ -179,6 +188,11 @@ processors: - json - zeek.smb_mapping.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-config.yml b/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-config.yml +++ b/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json b/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json index 2ea084127f0..fe35e44c794 100644 --- a/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json +++ b/packages/zeek/data_stream/smtp/_dev/test/pipeline/test-smtp.log-expected.json @@ -1,16 +1,6 @@ { "expected": [ { - "@timestamp": "2018-12-03T22:59:47.381Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "192.168.1.10", - "192.168.1.9" - ] - }, "destination": { "port": 25, "address": "192.168.1.9", @@ -30,16 +20,37 @@ "fuids": [] } }, - "tls": { - "established": true - }, "source": { "port": 33782, "address": "192.168.1.10", "ip": "192.168.1.10" }, + "tags": [ + "preserve_original_event" + ], + "network": { + "protocol": "smtp", + "community_id": "1:38H0puTqOoHT/5r2bKFUVSXifQw=", + "transport": "tcp" + }, + "@timestamp": "2018-12-03T22:59:47.381Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "192.168.1.10", + "192.168.1.9" + ] + }, + "tls": { + "established": true + }, "event": { - "ingested": "2021-04-23T19:56:26.337592400Z", + "ingested": "2021-06-08T08:26:51.218084900Z", + "original": "{\"ts\":1543877987.381899,\"uid\":\"CWWzPB3RjqhFf528c\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":33782,\"id.resp_h\":\"192.168.1.9\",\"id.resp_p\":25,\"trans_depth\":1,\"helo\":\"EXAMPLE.COM\",\"last_reply\":\"220 2.0.0 SMTP server ready\",\"path\":[\"192.168.1.9\"],\"tls\":true,\"fuids\":[],\"is_webmail\":false}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CWWzPB3RjqhFf528c", "category": [ "network" @@ -47,14 +58,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - }, - "network": { - "protocol": "smtp", - "community_id": "1:38H0puTqOoHT/5r2bKFUVSXifQw=", - "transport": "tcp" + ] } }, { @@ -87,6 +91,9 @@ "address": "192.168.1.10", "ip": "192.168.1.10" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "smtp", "community_id": "1:38H0puTqOoHT/5r2bKFUVSXifQw=", @@ -109,7 +116,7 @@ "established": true }, "event": { - "ingested": "2021-04-23T19:56:26.337598600Z", + "ingested": "2021-06-08T08:26:51.218094700Z", "original": "{\"ts\":1543877987.381899,\"uid\":\"CWWzPB3RjqhFf528c\",\"id.orig_h\":\"192.168.1.10\",\"id.orig_p\":33782,\"id.resp_h\":\"192.168.1.9\",\"id.resp_p\":25,\"trans_depth\":1,\"helo\":\"EXAMPLE.COM\",\"last_reply\":\"220 2.0.0 SMTP server ready\",\"path\":[\"192.168.1.9\"],\"tls\":true,\"fuids\":[],\"is_webmail\":false}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/smtp/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/smtp/agent/stream/log.yml.hbs b/packages/zeek/data_stream/smtp/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/smtp/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/smtp/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml index 1c5d19a70e8..4f8fee8fb80 100644 --- a/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/smtp/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: smtp - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.smtp - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.smtp @@ -207,6 +216,11 @@ processors: - json - zeek.smtp.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-config.yml b/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-config.yml +++ b/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json b/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json index 9f16b0ecc89..c7384b68948 100644 --- a/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json +++ b/packages/zeek/data_stream/snmp/_dev/test/pipeline/test-snmp.log-expected.json @@ -39,7 +39,10 @@ "ip": "192.168.1.2" }, "event": { - "ingested": "2021-04-23T19:56:26.428131500Z", + "ingested": "2021-06-08T08:27:19.195350200Z", + "original": "{\"ts\":1543877948.916584,\"uid\":\"CnKW1B4w9fpRa6Nkf2\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":59696,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":161,\"duration\":7.849924,\"version\":\"2c\",\"community\":\"public\",\"get_requests\":0,\"get_bulk_requests\":0,\"get_responses\":8,\"set_requests\":0,\"up_since\":1543631204.766508}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CnKW1B4w9fpRa6Nkf2", "category": [ "network" @@ -47,10 +50,11 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "snmp", "community_id": "1:X15ey/8/tEH+tlelK6P+GfgwBPc=", @@ -110,7 +114,10 @@ "ip": "184.105.139.67" }, "event": { - "ingested": "2021-04-23T19:56:26.428137900Z", + "ingested": "2021-06-08T08:27:19.195509Z", + "original": "{\"ts\":1617080496.400704,\"uid\":\"CxtWIB4ECPW89F8mSi\",\"id.orig_h\":\"184.105.139.67\",\"id.orig_p\":37533,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":161,\"duration\":0.0,\"version\":\"2c\",\"community\":\"public\",\"get_requests\":4,\"get_bulk_requests\":0,\"get_responses\":0,\"set_requests\":0}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CxtWIB4ECPW89F8mSi", "category": [ "network" @@ -118,10 +125,11 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "snmp", "community_id": "1:MUkMU0Syk5ccgUPSHnt5CrInr9E=", @@ -161,6 +169,9 @@ "address": "192.168.1.2", "ip": "192.168.1.2" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "snmp", "community_id": "1:X15ey/8/tEH+tlelK6P+GfgwBPc=", @@ -180,7 +191,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.428139800Z", + "ingested": "2021-06-08T08:27:19.195514300Z", "original": "{\"ts\":1543877948.916584,\"uid\":\"CnKW1B4w9fpRa6Nkf2\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":59696,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":161,\"duration\":7.849924,\"version\":\"2c\",\"community\":\"public\",\"get_requests\":0,\"get_bulk_requests\":0,\"get_responses\":8,\"set_requests\":0,\"up_since\":1543631204.766508}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/snmp/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/snmp/agent/stream/log.yml.hbs b/packages/zeek/data_stream/snmp/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/snmp/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/snmp/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml index 658a8afe719..5ee8e906a66 100644 --- a/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/snmp/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: snmp - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.snmp - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.snmp @@ -201,6 +210,11 @@ processors: - json - zeek.snmp.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-config.yml b/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-config.yml +++ b/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json b/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json index a384957c2ad..df9429912a4 100644 --- a/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json +++ b/packages/zeek/data_stream/socks/_dev/test/pipeline/test-socks.log-expected.json @@ -36,7 +36,8 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2021-04-23T19:56:26.527432600Z", + "ingested": "2021-06-08T08:27:43.745654100Z", + "original": "{\"ts\":1566508093.09494,\"uid\":\"Cmz4Cb4qCw1hGqYw1c\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":35368,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":8080,\"version\":5,\"status\":\"succeeded\",\"request.name\":\"www.google.com\",\"request_p\":443,\"bound.host\":\"0.0.0.0\",\"bound_p\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "Cmz4Cb4qCw1hGqYw1c", @@ -49,6 +50,9 @@ ], "outcome": "success" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "socks", "community_id": "1:1Hp/o0hOC62lAwrV+a0ZKDE3rrs=", @@ -86,6 +90,9 @@ "address": "127.0.0.1", "ip": "127.0.0.1" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "socks", "community_id": "1:1Hp/o0hOC62lAwrV+a0ZKDE3rrs=", @@ -104,7 +111,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.527438700Z", + "ingested": "2021-06-08T08:27:43.745663700Z", "original": "{\"ts\":1566508093.09494,\"uid\":\"Cmz4Cb4qCw1hGqYw1c\",\"id.orig_h\":\"127.0.0.1\",\"id.orig_p\":35368,\"id.resp_h\":\"127.0.0.1\",\"id.resp_p\":8080,\"version\":5,\"status\":\"succeeded\",\"request.name\":\"www.google.com\",\"request_p\":443,\"bound.host\":\"0.0.0.0\",\"bound_p\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/socks/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/socks/agent/stream/log.yml.hbs b/packages/zeek/data_stream/socks/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/socks/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/socks/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml index 28d93ec5f74..65962783771 100644 --- a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: socks - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.socks - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.socks @@ -219,6 +228,11 @@ processors: - json - zeek.socks.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-config.yml b/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-config.yml +++ b/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json b/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json index e6b0d3b250b..c108af7f48d 100644 --- a/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json +++ b/packages/zeek/data_stream/ssh/_dev/test/pipeline/test-ssh.log-expected.json @@ -42,7 +42,8 @@ "ip": "192.168.1.2" }, "event": { - "ingested": "2021-04-23T19:56:26.636866800Z", + "ingested": "2021-06-08T08:28:14.634513400Z", + "original": "{\"ts\":1562527532.904291,\"uid\":\"CajWfz1b3qnnWT0BU9\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":48380,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":22,\"version\":2,\"auth_success\":false,\"auth_attempts\":2,\"client\":\"SSH-2.0-OpenSSH_7.9p1 Ubuntu-10\",\"server\":\"SSH-2.0-OpenSSH_6.6.1p1 Debian-4~bpo70+1\",\"cipher_alg\":\"chacha20-poly1305@openssh.com\",\"mac_alg\":\"umac-64-etm@openssh.com\",\"compression_alg\":\"none\",\"kex_alg\":\"curve25519-sha256@libssh.org\",\"host_key_alg\":\"ecdsa-sha2-nistp256\",\"host_key\":\"86:71:ac:9c:35:1c:28:29:05:81:48:ec:66:67:de:bd\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", "id": "CajWfz1b3qnnWT0BU9", @@ -55,6 +56,9 @@ ], "outcome": "failure" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "ssh", "community_id": "1:42tg9bemt74qgrdvJOy2n5Veg4A=", @@ -108,7 +112,10 @@ "ip": "51.161.10.160" }, "event": { - "ingested": "2021-04-23T19:56:26.636872200Z", + "ingested": "2021-06-08T08:28:14.634517100Z", + "original": "{\"ts\":1617123417.413634,\"uid\":\"COXxsJ3dlSh6ECRYQj\",\"id.orig_h\":\"51.161.10.160\",\"id.orig_p\":38204,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":22,\"auth_attempts\":0,\"direction\":\"INBOUND\",\"client\":\"SSH-2.0-libssh-0.6.3\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "COXxsJ3dlSh6ECRYQj", "category": [ "network" @@ -116,10 +123,11 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "ssh", "community_id": "1:fEvwFYOBXBS6afWiC3Wd7zi4ym8=", @@ -173,7 +181,10 @@ "ip": "113.53.238.195" }, "event": { - "ingested": "2021-04-23T19:56:26.636874100Z", + "ingested": "2021-06-08T08:28:14.634521Z", + "original": "{\"ts\":1617123445.61524,\"uid\":\"CZPdXz1jfKSWzIDAeb\",\"id.orig_h\":\"113.53.238.195\",\"id.orig_p\":44164,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":22,\"auth_attempts\":0,\"direction\":\"INBOUND\",\"client\":\"SSH-2.0-libssh-0.6.3\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CZPdXz1jfKSWzIDAeb", "category": [ "network" @@ -181,10 +192,11 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "ssh", "community_id": "1:GsVj5goD0raV3RtUCa7RbCE4LM0=", @@ -238,7 +250,10 @@ "ip": "34.86.35.26" }, "event": { - "ingested": "2021-04-23T19:56:26.636875700Z", + "ingested": "2021-06-08T08:28:14.634524400Z", + "original": "{\"ts\":1617123450.957272,\"uid\":\"Cha1rs3OamonAZ4Nz6\",\"id.orig_h\":\"34.86.35.26\",\"id.orig_p\":33953,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":22,\"auth_attempts\":0,\"direction\":\"INBOUND\",\"client\":\"SSH-2.0-ZGrab ZGrab SSH Survey\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "Cha1rs3OamonAZ4Nz6", "category": [ "network" @@ -246,10 +261,11 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "ssh", "community_id": "1:hQmKiiCVA2EG4uaydkM5n4w8EZ4=", @@ -292,6 +308,9 @@ "address": "192.168.1.2", "ip": "192.168.1.2" }, + "tags": [ + "preserve_original_event" + ], "network": { "protocol": "ssh", "community_id": "1:42tg9bemt74qgrdvJOy2n5Veg4A=", @@ -311,7 +330,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:26.636877300Z", + "ingested": "2021-06-08T08:28:14.634530400Z", "original": "{\"ts\":1562527532.904291,\"uid\":\"CajWfz1b3qnnWT0BU9\",\"id.orig_h\":\"192.168.1.2\",\"id.orig_p\":48380,\"id.resp_h\":\"192.168.1.1\",\"id.resp_p\":22,\"version\":2,\"auth_success\":false,\"auth_attempts\":2,\"client\":\"SSH-2.0-OpenSSH_7.9p1 Ubuntu-10\",\"server\":\"SSH-2.0-OpenSSH_6.6.1p1 Debian-4~bpo70+1\",\"cipher_alg\":\"chacha20-poly1305@openssh.com\",\"mac_alg\":\"umac-64-etm@openssh.com\",\"compression_alg\":\"none\",\"kex_alg\":\"curve25519-sha256@libssh.org\",\"host_key_alg\":\"ecdsa-sha2-nistp256\",\"host_key\":\"86:71:ac:9c:35:1c:28:29:05:81:48:ec:66:67:de:bd\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ssh/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/ssh/agent/stream/log.yml.hbs b/packages/zeek/data_stream/ssh/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/ssh/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/ssh/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml index 5836500d09e..5edbf027ede 100644 --- a/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ssh/elasticsearch/ingest_pipeline/default.yml @@ -29,39 +29,48 @@ processors: - set: field: network.protocol value: ssh - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.ssh - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.ssh @@ -215,6 +224,11 @@ processors: - json - zeek.ssh.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-config.yml b/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-config.yml +++ b/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json b/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json index fc33a82bbcc..9c4968e09f6 100644 --- a/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json +++ b/packages/zeek/data_stream/ssl/_dev/test/pipeline/test-ssl.log-expected.json @@ -65,6 +65,9 @@ "address": "10.178.98.102", "ip": "10.178.98.102" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:1PMhYqOKBIyRAQeMbg/pWiJ198g=", "transport": "tcp" @@ -109,7 +112,10 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.790039800Z", + "ingested": "2021-06-08T08:28:52.700773800Z", + "original": "{\"ts\":1547688736.805088,\"uid\":\"CAOvs1BMFCX2Eh0Y3\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63199,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FebkbHWVCV8rEEEne\",\"F4BDY41MGUBT6URZMd\",\"FWlfEfiHVkv8evDL3\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CAOvs1BMFCX2Eh0Y3", "category": [ "network" @@ -117,9 +123,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -187,6 +191,9 @@ "address": "10.178.98.102", "ip": "10.178.98.102" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:zYbLmqRN6PLPB067HNAiAQISqvI=", "transport": "tcp" @@ -231,7 +238,10 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.790047400Z", + "ingested": "2021-06-08T08:28:52.700783500Z", + "original": "{\"ts\":1547688736.80509,\"uid\":\"C3mki91FnnNtm0u1ok\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63198,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"Fue9H32OmuitQk2zR\",\"FpbiBP215tk2xftxM6\",\"FEdROj1vUzTGw3BIUa\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C3mki91FnnNtm0u1ok", "category": [ "network" @@ -239,9 +249,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -309,6 +317,9 @@ "address": "10.178.98.102", "ip": "10.178.98.102" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:uvtDP+7asGjibinsGcMqvj9yAoc=", "transport": "tcp" @@ -353,7 +364,10 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.790049400Z", + "ingested": "2021-06-08T08:28:52.700788800Z", + "original": "{\"ts\":1547688736.805527,\"uid\":\"CfGBt82PzCXzHa0iek\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63197,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FiFLYv3UjeWyv2gcW\",\"FvSsiB1Xi816EMagI9\",\"FWpPS4mjGaAhTRXLf\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}{\"ts\":1602179457.352156,\"uid\":\"CK17Dl2SB8bZOVonSl\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":49228,\"id.resp_h\":\"192.168.50.1\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_RSA_WITH_AES_128_CBC_SHA256\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FOLwYQ6rs70bIMSf9\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=foo,OU=foo@bar,O=org,L=locality,C=LO\",\"issuer\":\"CN=CA,OU=CA@example.com,O=Example Corp,L=foo,C=HI\",\"validation_status\":\"self signed certificate\",\"ja3\":\"74927e242d6c3febf8cb9cab10a7f889\",\"ja3s\":\"80b3a14bccc8598a1f3bbe83e71f735f\",\"resp_certificate_sha1\":\"5dad8b55621b6b9c30679d9d61248dd132a83c94\",\"not_valid_before\":1562022421,\"not_valid_after\":1577748224}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CfGBt82PzCXzHa0iek", "category": [ "network" @@ -361,9 +375,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -407,6 +419,9 @@ "address": "10.156.0.2", "ip": "10.156.0.2" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:+VR+JlXwG/gg/ZUYvCR2rvevo0k=", "transport": "tcp" @@ -432,7 +447,10 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.790051300Z", + "ingested": "2021-06-08T08:28:52.700793900Z", + "original": "{\"ts\":1617091251.151303,\"uid\":\"CLQiVH1VcpvT3ruEak\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":52730,\"id.resp_h\":\"46.101.87.151\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\",\"resumed\":false,\"established\":false}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CLQiVH1VcpvT3ruEak", "category": [ "network" @@ -440,9 +458,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -482,6 +498,9 @@ "port": 52678, "ip": "35.195.125.46" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:V4vQxEzysZJXVi6XPnzaFJyex/U=", "transport": "tcp" @@ -504,7 +523,10 @@ "resumed": false }, "event": { - "ingested": "2021-04-23T19:56:26.790053600Z", + "ingested": "2021-06-08T08:28:52.700797700Z", + "original": "{\"ts\":1617090955.826099,\"uid\":\"CBiXOC4IqYxMv1xzf9\",\"id.orig_h\":\"35.195.125.46\",\"id.orig_p\":52678,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":443,\"server_name\":\"splunk-api.swiftcrypto.com\",\"resumed\":false,\"established\":false}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CBiXOC4IqYxMv1xzf9", "category": [ "network" @@ -512,9 +534,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -559,6 +579,9 @@ "port": 53368, "ip": "35.198.74.222" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:V9Qt7/8w9KL4Jtxsk2LcLXE5N8w=", "transport": "tcp" @@ -581,7 +604,10 @@ "resumed": false }, "event": { - "ingested": "2021-04-23T19:56:26.790055200Z", + "ingested": "2021-06-08T08:28:52.700803Z", + "original": "{\"ts\":1617091253.726384,\"uid\":\"C4jH9IqWGZwc1PPUh\",\"id.orig_h\":\"35.198.74.222\",\"id.orig_p\":53368,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":443,\"server_name\":\"tickets.swiftcrypto.com\",\"resumed\":false,\"established\":false}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C4jH9IqWGZwc1PPUh", "category": [ "network" @@ -589,9 +615,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -636,6 +660,9 @@ "port": 53382, "ip": "35.198.74.222" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:66wO2xP2DlLDi2zicRph+DuA9/E=", "transport": "tcp" @@ -658,7 +685,10 @@ "resumed": false }, "event": { - "ingested": "2021-04-23T19:56:26.790058300Z", + "ingested": "2021-06-08T08:28:52.700808500Z", + "original": "{\"ts\":1617091253.91861,\"uid\":\"CXVMSq6Dainy4WFN9\",\"id.orig_h\":\"35.198.74.222\",\"id.orig_p\":53382,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":443,\"server_name\":\"rundeck.swiftcrypto.com\",\"resumed\":false,\"established\":false}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CXVMSq6Dainy4WFN9", "category": [ "network" @@ -666,9 +696,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -720,6 +748,9 @@ "address": "10.156.0.2", "ip": "10.156.0.2" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:3IMDpvf8yf3uCJdX1xBFecnUlJQ=", "transport": "tcp" @@ -746,7 +777,10 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.798594300Z", + "ingested": "2021-06-08T08:28:52.700812100Z", + "original": "{\"ts\":1617091254.325291,\"uid\":\"CsgtQe4AikDZBsIM6k\",\"id.orig_h\":\"10.156.0.2\",\"id.orig_p\":55120,\"id.resp_h\":\"104.154.89.105\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\",\"curve\":\"secp256r1\",\"resumed\":false,\"established\":false,\"cert_chain_fuids\":[\"FeyRIk4nUtwwcUcnRf\"],\"client_cert_chain_fuids\":[],\"validation_status\":\"self signed certificate\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CsgtQe4AikDZBsIM6k", "category": [ "network" @@ -754,9 +788,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -796,6 +828,9 @@ "port": 53095, "ip": "35.195.125.46" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:4MfNex5Y2459jCDB+JNoM6rXM2U=", "transport": "tcp" @@ -818,7 +853,10 @@ "resumed": false }, "event": { - "ingested": "2021-04-23T19:56:26.798609400Z", + "ingested": "2021-06-08T08:28:52.700816100Z", + "original": "{\"ts\":1617091255.065602,\"uid\":\"CPGhJS3UPpcnR96NQc\",\"id.orig_h\":\"35.195.125.46\",\"id.orig_p\":53095,\"id.resp_h\":\"10.156.0.2\",\"id.resp_p\":443,\"server_name\":\"splunk-api.swiftcrypto.com\",\"resumed\":false,\"established\":false}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "CPGhJS3UPpcnR96NQc", "category": [ "network" @@ -826,9 +864,7 @@ "type": [ "connection", "protocol" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" + ] } }, { @@ -901,6 +937,9 @@ "address": "10.178.98.102", "ip": "10.178.98.102" }, + "tags": [ + "preserve_original_event" + ], "network": { "community_id": "1:uvtDP+7asGjibinsGcMqvj9yAoc=", "transport": "tcp" @@ -948,7 +987,7 @@ "version_protocol": "tls" }, "event": { - "ingested": "2021-04-23T19:56:26.798612200Z", + "ingested": "2021-06-08T08:28:52.700819600Z", "original": "{\"ts\":1547688736.805527,\"uid\":\"CfGBt82PzCXzHa0iek\",\"id.orig_h\":\"10.178.98.102\",\"id.orig_p\":63197,\"id.resp_h\":\"35.199.178.4\",\"id.resp_p\":9243,\"version\":\"TLSv12\",\"cipher\":\"TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\",\"curve\":\"secp256r1\",\"server_name\":\"dd625ffb4fc54735b281862aa1cd6cd4.us-west1.gcp.cloud.es.io\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FiFLYv3UjeWyv2gcW\",\"FvSsiB1Xi816EMagI9\",\"FWpPS4mjGaAhTRXLf\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=*.gcp.cloud.es.io,O=Elasticsearch\\u005c, Inc.,L=Mountain View,ST=California,C=US\",\"issuer\":\"CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US\",\"validation_status\":\"ok\"}{\"ts\":1602179457.352156,\"uid\":\"CK17Dl2SB8bZOVonSl\",\"id.orig_h\":\"10.0.0.1\",\"id.orig_p\":49228,\"id.resp_h\":\"192.168.50.1\",\"id.resp_p\":443,\"version\":\"TLSv12\",\"cipher\":\"TLS_RSA_WITH_AES_128_CBC_SHA256\",\"resumed\":false,\"established\":true,\"cert_chain_fuids\":[\"FOLwYQ6rs70bIMSf9\"],\"client_cert_chain_fuids\":[],\"subject\":\"CN=foo,OU=foo@bar,O=org,L=locality,C=LO\",\"issuer\":\"CN=CA,OU=CA@example.com,O=Example Corp,L=foo,C=HI\",\"validation_status\":\"self signed certificate\",\"ja3\":\"74927e242d6c3febf8cb9cab10a7f889\",\"ja3s\":\"80b3a14bccc8598a1f3bbe83e71f735f\",\"resp_certificate_sha1\":\"5dad8b55621b6b9c30679d9d61248dd132a83c94\",\"not_valid_before\":1562022421,\"not_valid_after\":1577748224}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/ssl/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/ssl/agent/stream/log.yml.hbs b/packages/zeek/data_stream/ssl/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/ssl/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/ssl/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml index a9f771539d9..0ca534c584a 100644 --- a/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/ssl/elasticsearch/ingest_pipeline/default.yml @@ -26,39 +26,48 @@ processors: - set: field: network.transport value: tcp - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.ssl - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.ssl @@ -516,6 +525,11 @@ processors: - json - zeek.ssl.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-config.yml b/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-config.yml +++ b/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json b/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json index a1511929ccf..783c5730bc9 100644 --- a/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json +++ b/packages/zeek/data_stream/stats/_dev/test/pipeline/test-stats.log-expected.json @@ -54,10 +54,14 @@ } }, "event": { - "ingested": "2021-04-23T19:56:27.258699Z", + "ingested": "2021-06-08T08:29:21.588567400Z", + "original": "{\"ts\":1476605878.714844,\"peer\":\"bro\",\"mem\":94,\"pkts_proc\":296,\"bytes_recv\":39674,\"events_proc\":723,\"events_queued\":728,\"active_tcp_conns\":1,\"active_udp_conns\":3,\"active_icmp_conns\":0,\"tcp_conns\":6,\"udp_conns\":36,\"icmp_conns\":2,\"timers\":797,\"active_timers\":38,\"files\":0,\"active_files\":0,\"dns_requests\":0,\"active_dns_requests\":0,\"reassem_tcp_size\":0,\"reassem_file_size\":0,\"reassem_frag_size\":0,\"reassem_unknown_size\":0}", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2016-10-16T08:17:58.714Z", @@ -121,11 +125,14 @@ } }, "event": { - "ingested": "2021-04-23T19:56:27.258707200Z", + "ingested": "2021-06-08T08:29:21.588571Z", "original": "{\"ts\":1476605878.714844,\"peer\":\"bro\",\"mem\":94,\"pkts_proc\":296,\"bytes_recv\":39674,\"events_proc\":723,\"events_queued\":728,\"active_tcp_conns\":1,\"active_udp_conns\":3,\"active_icmp_conns\":0,\"tcp_conns\":6,\"udp_conns\":36,\"icmp_conns\":2,\"timers\":797,\"active_timers\":38,\"files\":0,\"active_files\":0,\"dns_requests\":0,\"active_dns_requests\":0,\"reassem_tcp_size\":0,\"reassem_file_size\":0,\"reassem_frag_size\":0,\"reassem_unknown_size\":0}\r\n", "created": "2020-04-28T11:07:58.223Z", "kind": "metric" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/stats/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/stats/agent/stream/log.yml.hbs b/packages/zeek/data_stream/stats/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/stats/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/stats/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml index 2384293986f..06077bfa56b 100644 --- a/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/stats/elasticsearch/ingest_pipeline/default.yml @@ -14,39 +14,48 @@ processors: - set: field: ecs.version value: '1.9.0' - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.stats - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.stats @@ -162,6 +171,11 @@ processors: - message - json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/syslog/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/syslog/agent/stream/log.yml.hbs b/packages/zeek/data_stream/syslog/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/syslog/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/syslog/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml index c8bde430f54..38a24f6bd80 100644 --- a/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/syslog/elasticsearch/ingest_pipeline/default.yml @@ -17,39 +17,48 @@ processors: - set: field: network.protocol value: syslog - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.syslog - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.syslog @@ -182,6 +191,11 @@ processors: - message - json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-config.yml b/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-config.yml +++ b/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json b/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json index e597e9fbd89..e99f95474a3 100644 --- a/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json +++ b/packages/zeek/data_stream/traceroute/_dev/test/pipeline/test-traceroute.log-expected.json @@ -36,7 +36,8 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-04-23T19:56:27.356117800Z", + "ingested": "2021-06-08T08:30:19.374961800Z", + "original": "{\"ts\":1361916158.650605,\"src\":\"192.168.1.1\",\"dst\":\"8.8.8.8\",\"proto\":\"udp\"}", "category": [ "network" ], @@ -46,6 +47,9 @@ "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, + "tags": [ + "preserve_original_event" + ], "network": { "transport": "udp" } @@ -80,6 +84,9 @@ "address": "192.168.1.1", "ip": "192.168.1.1" }, + "tags": [ + "preserve_original_event" + ], "network": { "transport": "udp" }, @@ -97,7 +104,7 @@ "name": "Lees-MBP.localdomain" }, "event": { - "ingested": "2021-04-23T19:56:27.356124800Z", + "ingested": "2021-06-08T08:30:19.374968Z", "original": "{\"ts\":1361916158.650605,\"src\":\"192.168.1.1\",\"dst\":\"8.8.8.8\",\"proto\":\"udp\"}", "category": [ "network" diff --git a/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/traceroute/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/traceroute/agent/stream/log.yml.hbs b/packages/zeek/data_stream/traceroute/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/traceroute/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/traceroute/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml index 8ac8f6691f9..82b61bf8839 100644 --- a/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/traceroute/elasticsearch/ingest_pipeline/default.yml @@ -20,39 +20,48 @@ processors: - append: field: event.type value: info - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.traceroute - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.traceroute @@ -144,6 +153,11 @@ processors: - message - json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-config.yml b/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-config.yml +++ b/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json b/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json index 3ed54a8ec77..a63ce8ac4b0 100644 --- a/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json +++ b/packages/zeek/data_stream/tunnel/_dev/test/pipeline/test-tunnel.log-expected.json @@ -58,29 +58,23 @@ "ip": "132.16.146.79" }, "event": { + "ingested": "2021-06-08T08:30:42.586956300Z", + "original": "{\"ts\":1544405666.743509,\"id.orig_h\":\"132.16.146.79\",\"id.orig_p\":0,\"id.resp_h\":\"132.16.110.133\",\"id.resp_p\":8080,\"tunnel_type\":\"Tunnel::HTTP\",\"action\":\"Tunnel::DISCOVER\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "action": "Tunnel::DISCOVER", - "ingested": "2021-04-23T19:56:27.433629700Z", "category": [ "network" ], "type": [ "connection" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } - }, - { - "@timestamp": "2018-12-10T01:34:26.743Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "132.16.146.79", - "132.16.110.133" ] }, + "tags": [ + "preserve_original_event" + ] + }, + { "log": { "file": { "path": "/usr/local/var/log/zeek/tunnel.log" @@ -106,9 +100,6 @@ "port": 8080, "ip": "132.16.110.133" }, - "host": { - "name": "Lees-MBP.localdomain" - }, "zeek": { "tunnel": { "type": "Tunnel::HTTP", @@ -135,8 +126,24 @@ "port": 0, "ip": "132.16.146.79" }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2018-12-10T01:34:26.743Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "132.16.146.79", + "132.16.110.133" + ] + }, + "host": { + "name": "Lees-MBP.localdomain" + }, "event": { - "ingested": "2021-04-23T19:56:27.433636400Z", + "ingested": "2021-06-08T08:30:42.586960200Z", "original": "{\"ts\":1544405666.743509,\"id.orig_h\":\"132.16.146.79\",\"id.orig_p\":0,\"id.resp_h\":\"132.16.110.133\",\"id.resp_p\":8080,\"tunnel_type\":\"Tunnel::HTTP\",\"action\":\"Tunnel::DISCOVER\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/tunnel/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/tunnel/agent/stream/log.yml.hbs b/packages/zeek/data_stream/tunnel/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/tunnel/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/tunnel/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml index 0011461ea48..332867c427e 100644 --- a/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/tunnel/elasticsearch/ingest_pipeline/default.yml @@ -20,39 +20,48 @@ processors: - append: field: event.type value: connection - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.tunnel - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.tunnel @@ -178,6 +187,11 @@ processors: - json - zeek.tunnel.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-config.yml b/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-config.yml +++ b/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json b/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json index 8090a996de1..32b1f093585 100644 --- a/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json +++ b/packages/zeek/data_stream/weird/_dev/test/pipeline/test-weird.log-expected.json @@ -30,17 +30,21 @@ "ip": "192.168.1.1" }, "event": { - "ingested": "2021-04-23T19:56:27.521977200Z", + "ingested": "2021-06-08T08:31:05.100502800Z", + "original": "{\"ts\":1543877999.99354,\"uid\":\"C1ralPp062bkwWt4e\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":64521,\"id.resp_h\":\"192.168.1.2\",\"id.resp_p\":53,\"name\":\"dns_unmatched_reply\",\"notice\":false,\"peer\":\"worker-6\"}", + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", "id": "C1ralPp062bkwWt4e", "category": [ "network" ], "type": [ "info" - ], - "created": "2020-04-28T11:07:58.223Z", - "kind": "event" - } + ] + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2020-01-28T16:00:59.342Z", @@ -55,7 +59,8 @@ } }, "event": { - "ingested": "2021-04-23T19:56:27.521983300Z", + "ingested": "2021-06-08T08:31:05.100511900Z", + "original": "{\"ts\":1580227259.342809,\"name\":\"non_ip_packet_in_ethernet\",\"notice\":false,\"peer\":\"ens3f1-4\"}", "category": [ "network" ], @@ -64,19 +69,12 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "tags": [ + "preserve_original_event" + ] }, { - "@timestamp": "2018-12-03T22:59:59.993Z", - "ecs": { - "version": "1.9.0" - }, - "related": { - "ip": [ - "192.168.1.1", - "192.168.1.2" - ] - }, "log": { "file": { "path": "/usr/local/var/log/zeek/weird.log" @@ -87,9 +85,6 @@ "address": "192.168.1.2", "ip": "192.168.1.2" }, - "host": { - "name": "Lees-MBP.localdomain" - }, "zeek": { "weird": { "name": "dns_unmatched_reply", @@ -103,8 +98,24 @@ "address": "192.168.1.1", "ip": "192.168.1.1" }, + "tags": [ + "preserve_original_event" + ], + "@timestamp": "2018-12-03T22:59:59.993Z", + "ecs": { + "version": "1.9.0" + }, + "related": { + "ip": [ + "192.168.1.1", + "192.168.1.2" + ] + }, + "host": { + "name": "Lees-MBP.localdomain" + }, "event": { - "ingested": "2021-04-23T19:56:27.521985200Z", + "ingested": "2021-06-08T08:31:05.100518900Z", "original": "{\"ts\":1543877999.99354,\"uid\":\"C1ralPp062bkwWt4e\",\"id.orig_h\":\"192.168.1.1\",\"id.orig_p\":64521,\"id.resp_h\":\"192.168.1.2\",\"id.resp_p\":53,\"name\":\"dns_unmatched_reply\",\"notice\":false,\"peer\":\"worker-6\"}", "created": "2020-04-28T11:07:58.223Z", "kind": "event", diff --git a/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs index 59300114a90..c9c1e585e8f 100644 --- a/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/weird/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/weird/agent/stream/log.yml.hbs b/packages/zeek/data_stream/weird/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/weird/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/weird/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml index fd3a9dd66b7..41016e1a898 100644 --- a/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/weird/elasticsearch/ingest_pipeline/default.yml @@ -20,39 +20,48 @@ processors: - append: field: event.type value: info - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.weird - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.weird @@ -176,6 +185,11 @@ processors: - json - zeek.weird.id ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-config.yml b/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-config.yml index 9827c05de7c..3cabcf9fb82 100644 --- a/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-config.yml +++ b/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-config.yml @@ -2,3 +2,5 @@ dynamic_fields: event.ingested: ".*" fields: "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json b/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json index f310ae9e87e..af469869c36 100644 --- a/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json +++ b/packages/zeek/data_stream/x509/_dev/test/pipeline/test-x509.log-expected.json @@ -210,14 +210,18 @@ "session_id": "FxZ6gZ3YR6vFlIocq3" }, "event": { - "ingested": "2021-04-23T19:56:27.603067300Z", + "ingested": "2021-06-08T08:31:44.544051500Z", + "original": "{\"ts\":1543867200.143484,\"id\":\"FxZ6gZ3YR6vFlIocq3\",\"certificate.version\":3,\"certificate.serial\":\"2D00003299D7071DB7D1708A42000000003299\",\"certificate.subject\":\"CN=www.bing.com\",\"certificate.issuer\":\"CN=Microsoft IT TLS CA 5,OU=Microsoft IT,O=Microsoft Corporation,L=Redmond,ST=Washington,C=US\",\"certificate.not_valid_before\":1500572828.0,\"certificate.not_valid_after\":1562780828.0,\"certificate.key_alg\":\"rsaEncryption\",\"certificate.sig_alg\":\"sha256WithRSAEncryption\",\"certificate.key_type\":\"rsa\",\"certificate.key_length\":2048,\"certificate.exponent\":\"65537\",\"san.dns\":[\"www.bing.com\",\"dict.bing.com.cn\",\"*.platform.bing.com\",\"*.bing.com\",\"bing.com\",\"ieonline.microsoft.com\",\"*.windowssearch.com\",\"cn.ieonline.microsoft.com\",\"*.origin.bing.com\",\"*.mm.bing.net\",\"*.api.bing.com\",\"ecn.dev.virtualearth.net\",\"*.cn.bing.net\",\"*.cn.bing.com\",\"ssl-api.bing.com\",\"ssl-api.bing.net\",\"*.api.bing.net\",\"*.bingapis.com\",\"bingsandbox.com\",\"feedback.microsoft.com\",\"insertmedia.bing.office.net\",\"r.bat.bing.com\",\"*.r.bat.bing.com\",\"*.dict.bing.com.cn\",\"*.dict.bing.com\",\"*.ssl.bing.com\",\"*.appex.bing.com\",\"*.platform.cn.bing.com\",\"wp.m.bing.com\",\"*.m.bing.com\",\"global.bing.com\",\"windowssearch.com\",\"search.msn.com\",\"*.bingsandbox.com\",\"*.api.tiles.ditu.live.com\",\"*.ditu.live.com\",\"*.t0.tiles.ditu.live.com\",\"*.t1.tiles.ditu.live.com\",\"*.t2.tiles.ditu.live.com\",\"*.t3.tiles.ditu.live.com\",\"*.tiles.ditu.live.com\",\"3d.live.com\",\"api.search.live.com\",\"beta.search.live.com\",\"cnweb.search.live.com\",\"dev.live.com\",\"ditu.live.com\",\"farecast.live.com\",\"image.live.com\",\"images.live.com\",\"local.live.com.au\",\"localsearch.live.com\",\"ls4d.search.live.com\",\"mail.live.com\",\"mapindia.live.com\",\"local.live.com\",\"maps.live.com\",\"maps.live.com.au\",\"mindia.live.com\",\"news.live.com\",\"origin.cnweb.search.live.com\",\"preview.local.live.com\",\"search.live.com\",\"test.maps.live.com\",\"video.live.com\",\"videos.live.com\",\"virtualearth.live.com\",\"wap.live.com\",\"webmaster.live.com\",\"webmasters.live.com\",\"www.local.live.com.au\",\"www.maps.live.com.au\"]}", "id": "FxZ6gZ3YR6vFlIocq3", "type": [ "info" ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "tags": [ + "preserve_original_event" + ] }, { "@timestamp": "2018-12-03T20:00:00.143Z", @@ -437,7 +441,7 @@ "session_id": "FxZ6gZ3YR6vFlIocq3" }, "event": { - "ingested": "2021-04-23T19:56:27.603073Z", + "ingested": "2021-06-08T08:31:44.544059900Z", "original": "{\"ts\":1543867200.143484,\"id\":\"FxZ6gZ3YR6vFlIocq3\",\"certificate.version\":3,\"certificate.serial\":\"2D00003299D7071DB7D1708A42000000003299\",\"certificate.subject\":\"CN=www.bing.com\",\"certificate.issuer\":\"CN=Microsoft IT TLS CA 5,OU=Microsoft IT,O=Microsoft Corporation,L=Redmond,ST=Washington,C=US\",\"certificate.not_valid_before\":1500572828.0,\"certificate.not_valid_after\":1562780828.0,\"certificate.key_alg\":\"rsaEncryption\",\"certificate.sig_alg\":\"sha256WithRSAEncryption\",\"certificate.key_type\":\"rsa\",\"certificate.key_length\":2048,\"certificate.exponent\":\"65537\",\"san.dns\":[\"www.bing.com\",\"dict.bing.com.cn\",\"*.platform.bing.com\",\"*.bing.com\",\"bing.com\",\"ieonline.microsoft.com\",\"*.windowssearch.com\",\"cn.ieonline.microsoft.com\",\"*.origin.bing.com\",\"*.mm.bing.net\",\"*.api.bing.com\",\"ecn.dev.virtualearth.net\",\"*.cn.bing.net\",\"*.cn.bing.com\",\"ssl-api.bing.com\",\"ssl-api.bing.net\",\"*.api.bing.net\",\"*.bingapis.com\",\"bingsandbox.com\",\"feedback.microsoft.com\",\"insertmedia.bing.office.net\",\"r.bat.bing.com\",\"*.r.bat.bing.com\",\"*.dict.bing.com.cn\",\"*.dict.bing.com\",\"*.ssl.bing.com\",\"*.appex.bing.com\",\"*.platform.cn.bing.com\",\"wp.m.bing.com\",\"*.m.bing.com\",\"global.bing.com\",\"windowssearch.com\",\"search.msn.com\",\"*.bingsandbox.com\",\"*.api.tiles.ditu.live.com\",\"*.ditu.live.com\",\"*.t0.tiles.ditu.live.com\",\"*.t1.tiles.ditu.live.com\",\"*.t2.tiles.ditu.live.com\",\"*.t3.tiles.ditu.live.com\",\"*.tiles.ditu.live.com\",\"3d.live.com\",\"api.search.live.com\",\"beta.search.live.com\",\"cnweb.search.live.com\",\"dev.live.com\",\"ditu.live.com\",\"farecast.live.com\",\"image.live.com\",\"images.live.com\",\"local.live.com.au\",\"localsearch.live.com\",\"ls4d.search.live.com\",\"mail.live.com\",\"mapindia.live.com\",\"local.live.com\",\"maps.live.com\",\"maps.live.com.au\",\"mindia.live.com\",\"news.live.com\",\"origin.cnweb.search.live.com\",\"preview.local.live.com\",\"search.live.com\",\"test.maps.live.com\",\"video.live.com\",\"videos.live.com\",\"virtualearth.live.com\",\"wap.live.com\",\"webmaster.live.com\",\"webmasters.live.com\",\"www.local.live.com.au\",\"www.maps.live.com.au\"]}", "id": "FxZ6gZ3YR6vFlIocq3", "type": [ @@ -445,7 +449,10 @@ ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" - } + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs index 570c6fc2fd7..6b52c17ceee 100644 --- a/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs +++ b/packages/zeek/data_stream/x509/agent/stream/httpjson.yml.hbs @@ -39,3 +39,6 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} diff --git a/packages/zeek/data_stream/x509/agent/stream/log.yml.hbs b/packages/zeek/data_stream/x509/agent/stream/log.yml.hbs index 7fdc0e30563..0972ba9cd56 100644 --- a/packages/zeek/data_stream/x509/agent/stream/log.yml.hbs +++ b/packages/zeek/data_stream/x509/agent/stream/log.yml.hbs @@ -9,6 +9,9 @@ tags: {{#each tags as |tag i|}} - {{tag}} {{/each}} +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true {{/contains}} diff --git a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml index 2d72e70ca6f..913f9a19ef4 100644 --- a/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/x509/elasticsearch/ingest_pipeline/default.yml @@ -17,39 +17,48 @@ processors: - append: field: event.type value: info - - json: + - rename: field: message - target_field: json - ignore_failure: true - if: ctx?.message != null + target_field: event.original + - json: + field: event.original + target_field: _temp_.json - drop: - if: 'ctx?.json?.result == null && ctx?.json?.ts == null' - - rename: - field: json - target_field: zeek.x509 - if: ctx?.json?.ts != null + description: Drop if no Splunk or log data present. + if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' +# Splunk specific parsing start - fingerprint: fields: - - json.result._cd - - json.result._indextime - - json.result._raw - - json.result._time - - json.result.host - - json.result.source + - _temp_.json.result._cd + - _temp_.json.result._indextime + - _temp_.json.result._raw + - _temp_.json.result._time + - _temp_.json.result.host + - _temp_.json.result.source target_field: '_id' - if: 'ctx?.json?.result != null && ctx?.json?.ts == null' - - rename: - field: json.result._raw - target_field: event.original + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - remove: + field: event.original ignore_missing: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' + - set: + field: event.original + copy_from: _temp_.json.result._raw + ignore_empty_value: true + ignore_failure: true + if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null' - rename: - field: json.result.host + field: _temp_.json.result.host target_field: host.name ignore_missing: true - rename: - field: json.result.source + field: _temp_.json.result.source target_field: log.file.path ignore_missing: true + - remove: + field: _temp_ + ignore_missing: true +# Splunk parsing end - json: field: event.original target_field: zeek.x509 @@ -437,6 +446,11 @@ processors: - message - json ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index b3f5471d4d7..2b1567857c5 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek -version: 0.7.2 +version: 0.7.3 release: beta description: Zeek Integration type: integration @@ -39,6 +39,14 @@ policy_templates: - /var/log/bro/current - /opt/zeek/logs/current - /usr/local/var/spool/zeek + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false - type: httpjson title: Collect logs from third-party REST API (experimental) description: Collect logs from third-party REST API (experimental) @@ -60,6 +68,14 @@ policy_templates: title: Splunk REST API Password required: true show_user: true + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false - name: ssl type: yaml title: SSL Configuration