Skip to content

Commit

Permalink
Sync with zeek package
Browse files Browse the repository at this point in the history
- connection
  + remove redundant source.ip & destination.ip mapping
  + add null check for append to related.ip
  + remove duplicate setting of event.kind & event.category
- dce_rpc
  + add null check for append to related.ip
- dnp3
  + add append to related.ip
- dns
  + move conversion of zeek.dns.ts to @timestamp to ingest pipeline
  + add event.created to ingest pipeline
- intel
  + move conversion of zeek.intel.ts to @timestamp to ingest pipeline
- socks
  + change processor for event.outcome from append to set
- ssl
  + fix so event.type is correctly set
  + fix quoting of = in ingest pipeline
- x509
  + convert pipeline to yaml format
  • Loading branch information
leehinman committed Aug 19, 2020
1 parent 5f42d21 commit 22aeb09
Show file tree
Hide file tree
Showing 18 changed files with 184 additions and 283 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Improve validation checks for Azure configuration {issue}20369[20369] {pull}20389[20389]
- Fix event.kind for system/syslog pipeline {issue}20365[20365] {pull}20390[20390]
- Clone value when copy fields in processors to avoid crash. {issue}19206[19206] {pull}20500[20500]
- Fix event.type for zeek/ssl and duplicate event.category for zeek/connection {pull}20696[20696]

*Heartbeat*

Expand Down
18 changes: 4 additions & 14 deletions x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,14 @@ processors:
- script:
source: if (ctx.zeek.connection.local_resp) ctx.tags.add("local_resp");
if: ctx.zeek.connection.local_resp != null
- set:
field: source.ip
value: '{{source.address}}'
- append:
field: related.ip
value: '{{source.address}}'
- set:
field: destination.ip
value: '{{destination.address}}'
value: '{{source.ip}}'
if: ctx?.source?.ip != null
- append:
field: related.ip
value: '{{destination.address}}'
value: '{{destination.ip}}'
if: ctx?.destination?.ip != null
- script:
source: ctx.network.packets = ctx.source.packets + ctx.destination.packets
ignore_failure: true
Expand Down Expand Up @@ -101,12 +97,6 @@ processors:
field: destination.as.organization_name
target_field: destination.as.organization.name
ignore_missing: true
- set:
field: event.kind
value: event
- append:
field: event.category
value: network
- script:
params:
S0:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
"destination.packets": 1,
"destination.port": 53,
"event.category": [
"network",
"network"
],
"event.dataset": "zeek.connection",
Expand Down Expand Up @@ -66,7 +65,6 @@
"destination.packets": 1,
"destination.port": 53,
"event.category": [
"network",
"network"
],
"event.dataset": "zeek.connection",
Expand Down Expand Up @@ -124,7 +122,6 @@
"destination.packets": 1,
"destination.port": 53,
"event.category": [
"network",
"network"
],
"event.dataset": "zeek.connection",
Expand Down Expand Up @@ -180,7 +177,6 @@
"destination.ip": "198.51.100.249",
"destination.packets": 0,
"event.category": [
"network",
"network"
],
"event.dataset": "zeek.connection",
Expand Down
2 changes: 2 additions & 0 deletions x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ processors:
- append:
field: related.ip
value: '{{source.ip}}'
if: ctx?.source?.ip != null
- geoip:
field: source.ip
target_field: source.geo
Expand All @@ -37,6 +38,7 @@ processors:
- append:
field: related.ip
value: '{{destination.ip}}'
if: ctx?.destination?.ip != null
- geoip:
field: destination.ip
target_field: destination.geo
Expand Down
8 changes: 8 additions & 0 deletions x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,14 @@ processors:
- lowercase:
field: event.action
ignore_missing: true
- append:
field: related.ip
value: '{{source.ip}}'
if: ctx?.source?.ip != null
- append:
field: related.ip
value: '{{destination.ip}}'
if: ctx?.destination?.ip != null
- geoip:
field: destination.ip
target_field: destination.geo
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@
"network.community_id": "1:E57Z1w3RrSdR+fi6rSZblbQVhzY=",
"network.protocol": "dnp3",
"network.transport": "tcp",
"related.ip": [
"127.0.0.1",
"127.0.0.1"
],
"service.type": "zeek",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
7 changes: 0 additions & 7 deletions x-pack/filebeat/module/zeek/dns/config/dns.yml
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,6 @@ processors:
- {from: source.address, to: source.ip, type: ip}
- {from: destination.address, to: destination.ip, type: ip}
- {from: zeek.session_id, to: event.id}
- {from: '@timestamp', to: event.created}
- {from: zeek.dns.trans_id, to: dns.id}
- {from: zeek.dns.query, to: dns.question.name}
- {from: zeek.dns.qtype_name, to: dns.question.type}
Expand All @@ -197,18 +196,12 @@ processors:
{{ if .community_id }}
- community_id:
{{ end }}
- timestamp:
ignore_missing: true
field: zeek.dns.ts
layouts:
- UNIX
- drop_fields:
ignore_missing: true
fields:
- zeek.dns.Z
- zeek.dns.auth
- zeek.dns.addl
- zeek.dns.ts
- add_fields:
target: ''
fields:
Expand Down
9 changes: 9 additions & 0 deletions x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,15 @@ processors:
- set:
field: event.ingested
value: '{{_ingest.timestamp}}'
- set:
field: event.created
value: '{{@timestamp}}'
- date:
field: zeek.dns.ts
formats:
- UNIX
- remove:
field: zeek.dns.ts

# IP Geolocation Lookup
- geoip:
Expand Down
5 changes: 0 additions & 5 deletions x-pack/filebeat/module/zeek/intel/config/intel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,6 @@ processors:
- decode_json_fields:
fields: [event.original]
target: zeek.intel
- timestamp:
field: zeek.intel.ts
layouts: [UNIX]
test: "1573030980.989353"
- convert:
ignore_missing: true
fields:
Expand Down Expand Up @@ -57,7 +53,6 @@ processors:
- drop_fields:
ignore_missing: true
fields:
- zeek.intel.ts
- zeek.intel.id.orig_h
- zeek.intel.id.orig_p
- zeek.intel.id.resp_h
Expand Down
7 changes: 6 additions & 1 deletion x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@ processors:
- set:
field: event.created
value: '{{@timestamp}}'

- date:
field: zeek.intel.ts
formats:
- UNIX
- remove:
field: zeek.intel.ts
# IP Geolocation Lookup
- geoip:
if: ctx.source?.geo == null
Expand Down
4 changes: 2 additions & 2 deletions x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,11 @@ processors:
field: event.type
value: error
if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status != 'succeeded'"
- append:
- set:
field: event.outcome
value: success
if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status == 'succeeded'"
- append:
- set:
field: event.outcome
value: failure
if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status != 'succeeded'"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@
"event.id": "Cmz4Cb4qCw1hGqYw1c",
"event.kind": "event",
"event.module": "zeek",
"event.outcome": [
"success"
],
"event.outcome": "success",
"event.type": [
"connection",
"protocol"
Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/module/zeek/ssl/config/ssl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ processors:
kind: event
category:
- network
kind:
type:
- connection
- protocol
{{ if .community_id }}
Expand Down
8 changes: 4 additions & 4 deletions x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ processors:
- kv:
field: zeek.ssl.issuer
field_split: ','
value_split: =
value_split: '='
target_field: zeek.ssl.server.issuer
ignore_missing: true
- rename:
Expand Down Expand Up @@ -104,7 +104,7 @@ processors:
- kv:
field: zeek.ssl.subject
field_split: ','
value_split: =
value_split: '='
target_field: zeek.ssl.server.subject
ignore_missing: true
- remove:
Expand Down Expand Up @@ -142,7 +142,7 @@ processors:
- kv:
field: zeek.ssl.client_issuer
field_split: ','
value_split: =
value_split: '='
target_field: zeek.ssl.client.issuer
ignore_missing: true
- rename:
Expand Down Expand Up @@ -181,7 +181,7 @@ processors:
- kv:
field: zeek.ssl.client_subject
field_split: ','
value_split: =
value_split: '='
target_field: zeek.ssl.client.subject
ignore_missing: true
- remove:
Expand Down
10 changes: 6 additions & 4 deletions x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@
],
"event.dataset": "zeek.ssl",
"event.id": "CAOvs1BMFCX2Eh0Y3",
"event.kind": [
"event.kind": "event",
"event.module": "zeek",
"event.type": [
"connection",
"protocol"
],
"event.module": "zeek",
"fileset.name": "ssl",
"input.type": "log",
"log.offset": 0,
Expand Down Expand Up @@ -90,11 +91,12 @@
],
"event.dataset": "zeek.ssl",
"event.id": "C3mki91FnnNtm0u1ok",
"event.kind": [
"event.kind": "event",
"event.module": "zeek",
"event.type": [
"connection",
"protocol"
],
"event.module": "zeek",
"fileset.name": "ssl",
"input.type": "log",
"log.offset": 635,
Expand Down
Loading

0 comments on commit 22aeb09

Please sign in to comment.