Skip to content

Commit

Permalink
Drop pkt_dstaddr and pkt_srcaddr when equals to "-" (elastic#22721)
Browse files Browse the repository at this point in the history
* Add painless script to remove all empty fields

(cherry picked from commit 24a4da8)
  • Loading branch information
kaiyan-sheng authored and ChrsMark committed Dec 1, 2020
1 parent 711762b commit 510e8da
Show file tree
Hide file tree
Showing 6 changed files with 90 additions and 3 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696]
- Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361]
- Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377]
- Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]

*Heartbeat*

Expand Down
26 changes: 26 additions & 0 deletions x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,32 @@ processors:
field: ["aws.vpcflow.start", "aws.vpcflow.end"]
ignore_missing: true

- script:
lang: painless
ignore_failure: true
if: ctx?.aws != null
source: >-
void handleMap(Map map) {
for (def x : map.values()) {
if (x instanceof Map) {
handleMap(x);
} else if (x instanceof List) {
handleList(x);
}
}
map.values().removeIf(v -> v instanceof String && v == "-");
}
void handleList(List list) {
for (def x : list) {
if (x instanceof Map) {
handleMap(x);
} else if (x instanceof List) {
handleList(x);
}
}
}
handleMap(ctx.aws);
# IP Geolocation Lookup
- geoip:
field: source.ip
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
[
{
"aws.vpcflow.instance_id": "-",
"aws.vpcflow.interface_id": "eni-1235b8ca123456789",
"aws.vpcflow.pkt_dstaddr": "203.0.113.5",
"aws.vpcflow.pkt_srcaddr": "10.0.1.5",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
{
"@timestamp": "2015-05-10T18:02:14.000Z",
"aws.vpcflow.account_id": "123456789010",
"aws.vpcflow.action": "-",
"aws.vpcflow.interface_id": "eni-1235b8ca123456789",
"aws.vpcflow.log_status": "NODATA",
"aws.vpcflow.version": "2",
Expand All @@ -27,7 +26,6 @@
{
"@timestamp": "2015-05-10T18:02:14.000Z",
"aws.vpcflow.account_id": "123456789010",
"aws.vpcflow.action": "-",
"aws.vpcflow.interface_id": "eni-11111111aaaaaaaaa",
"aws.vpcflow.log_status": "SKIPDATA",
"aws.vpcflow.version": "2",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA

version vpc-id subnet-id instance-id interface-id account-id type srcaddr dstaddr srcport dstport pkt-srcaddr pkt-dstaddr protocol bytes packets start end action tcp-flags log-status
3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
[
{
"@timestamp": "2019-08-26T19:48:53.000Z",
"aws.vpcflow.account_id": "123456789010",
"aws.vpcflow.instance_id": "i-01234567890123456",
"aws.vpcflow.interface_id": "eni-1235b8ca123456789",
"aws.vpcflow.log_status": "SKIPDATA",
"aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678",
"aws.vpcflow.version": "3",
"aws.vpcflow.vpc_id": "vpc-abcdefab012345678",
"cloud.account.id": "123456789010",
"cloud.instance.id": "i-01234567890123456",
"cloud.provider": "aws",
"event.category": "network_traffic",
"event.dataset": "aws.vpcflow",
"event.end": "2019-08-26T19:48:53.000Z",
"event.kind": "event",
"event.module": "aws",
"event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - SKIPDATA",
"event.start": "2019-08-26T19:47:55.000Z",
"event.type": "flow",
"fileset.name": "vpcflow",
"input.type": "log",
"log.offset": 183,
"service.type": "aws",
"tags": [
"forwarded"
]
},
{
"@timestamp": "2019-08-26T19:48:53.000Z",
"aws.vpcflow.account_id": "123456789010",
"aws.vpcflow.instance_id": "i-01234567890123456",
"aws.vpcflow.interface_id": "eni-1235b8ca123456789",
"aws.vpcflow.log_status": "NODATA",
"aws.vpcflow.subnet_id": "subnet-aaaaaaaa012345678",
"aws.vpcflow.version": "3",
"aws.vpcflow.vpc_id": "vpc-abcdefab012345678",
"cloud.account.id": "123456789010",
"cloud.instance.id": "i-01234567890123456",
"cloud.provider": "aws",
"event.category": "network_traffic",
"event.dataset": "aws.vpcflow",
"event.end": "2019-08-26T19:48:53.000Z",
"event.kind": "event",
"event.module": "aws",
"event.original": "3 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-01234567890123456 eni-1235b8ca123456789 123456789010 - - - - - - - - - - 1566848875 1566848933 - - NODATA",
"event.start": "2019-08-26T19:47:55.000Z",
"event.type": "flow",
"fileset.name": "vpcflow",
"input.type": "log",
"log.offset": 526,
"service.type": "aws",
"tags": [
"forwarded"
]
}
]

0 comments on commit 510e8da

Please sign in to comment.