diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cdd5a87b34a..be26ee7e6b2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -198,6 +198,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix syslog RFC 5424 parsing in the CheckPoint module. {pull}21854[21854] - Fix incorrect connection state mapping in zeek connection pipeline. {pull}22151[22151] {issue}22149[22149] - Fix missing variable when loading aws pipelines. {pull}22645[22645] +- Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361] +- Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696] +- Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377] +- Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716] *Heartbeat* diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index bd9b1d32769..a8a6e5ae726 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -28,6 +28,32 @@ processors: field: ["aws.vpcflow.start", "aws.vpcflow.end"] ignore_missing: true + - script: + lang: painless + ignore_failure: true + if: ctx?.aws != null + source: >- + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v instanceof String && v == "-"); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx.aws); + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json index d508bd63479..6b9e4382bb5 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/custom-nat-gateway.log-expected.json @@ -1,6 +1,5 @@ [ { - "aws.vpcflow.instance_id": "-", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.pkt_dstaddr": "203.0.113.5", "aws.vpcflow.pkt_srcaddr": "10.0.1.5", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json index 22705d87101..e8224ee08b1 100644 --- a/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json +++ b/x-pack/filebeat/module/aws/vpcflow/test/no-data-skip-data.log-expected.json @@ -2,7 +2,6 @@ { "@timestamp": "2015-05-10T18:02:14.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.interface_id": "eni-1235b8ca123456789", "aws.vpcflow.log_status": "NODATA", "aws.vpcflow.version": "2", @@ -27,7 +26,6 @@ { "@timestamp": "2015-05-10T18:02:14.000Z", "aws.vpcflow.account_id": "123456789010", - "aws.vpcflow.action": "-", "aws.vpcflow.interface_id": "eni-11111111aaaaaaaaa", "aws.vpcflow.log_status": "SKIPDATA", "aws.vpcflow.version": "2", diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log new file mode 100644 index 00000000000..2ce24460ff9 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log @@ -0,0 +1,5 @@ +version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA + +version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status +3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA diff --git a/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json new file mode 100644 index 00000000000..b28207021b6 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/tcp-flag-sequence-skip-data.log-expected.json @@ -0,0 +1,58 @@ +[ + { + "@timestamp": "2019-08-26T19:48:53.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "SKIPDATA", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA", + "event.start": "2019-08-26T19:47:55.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 183, + "service.type": "aws", + "tags": [ + "forwarded" + ] + }, + { + "@timestamp": "2019-08-26T19:48:53.000Z", + "aws.vpcflow.account_id": "123456789010", + "aws.vpcflow.instance_id": "i-01234567890123456", + "aws.vpcflow.interface_id": "eni-1235b8ca123456789", + "aws.vpcflow.log_status": "NODATA", + "aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678", + "aws.vpcflow.version": "3", + "aws.vpcflow.vpc_id": "vpc-abcdefab012345678", + "cloud.account.id": "123456789010", + "cloud.instance.id": "i-01234567890123456", + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.end": "2019-08-26T19:48:53.000Z", + "event.kind": "event", + "event.module": "aws", + "event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA", + "event.start": "2019-08-26T19:47:55.000Z", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 526, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file