Sync with zeek package (elastic#20696)

- connection + remove redundant source.ip & destination.ip mapping + add null check for append to related.ip + remove duplicate setting of event.kind & event.category - dce_rpc + add null check for append to related.ip - dnp3 + add append to related.ip - dns + move conversion of zeek.dns.ts to @timestamp to ingest pipeline + add event.created to ingest pipeline - intel + move conversion of zeek.intel.ts to @timestamp to ingest pipeline - socks + change processor for event.outcome from append to set - ssl + fix so event.type is correctly set + fix quoting of = in ingest pipeline - x509 + convert pipeline to yaml format
leehinman · Aug 20, 2020 · 7de72d6 · 7de72d6
1 parent 98d3925
commit 7de72d6
Show file tree

Hide file tree

Showing 18 changed files with 184 additions and 283 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -248,6 +248,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve validation checks for Azure configuration {issue}20369[20369] {pull}20389[20389]
 - Fix event.kind for system/syslog pipeline {issue}20365[20365] {pull}20390[20390]
 - Clone value when copy fields in processors to avoid crash. {issue}19206[19206] {pull}20500[20500]
+- Fix event.type for zeek/ssl and duplicate event.category for zeek/connection {pull}20696[20696]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml
@@ -30,18 +30,14 @@ processors:
 - script:
     source: if (ctx.zeek.connection.local_resp) ctx.tags.add("local_resp");
     if: ctx.zeek.connection.local_resp != null
-- set:
-    field: source.ip
-    value: '{{source.address}}'
 - append:
     field: related.ip
-    value: '{{source.address}}'
-- set:
-    field: destination.ip
-    value: '{{destination.address}}'
+    value: '{{source.ip}}'
+    if: ctx?.source?.ip != null
 - append:
     field: related.ip
-    value: '{{destination.address}}'
+    value: '{{destination.ip}}'
+    if: ctx?.destination?.ip != null
 - script:
     source: ctx.network.packets = ctx.source.packets + ctx.destination.packets
     ignore_failure: true
@@ -101,12 +97,6 @@ processors:
     field: destination.as.organization_name
     target_field: destination.as.organization.name
     ignore_missing: true
-- set:
-    field: event.kind
-    value: event
-- append:
-    field: event.category
-    value: network
 - script:
     params:
       S0:

diff --git a/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json b/x-pack/filebeat/module/zeek/connection/test/connection-json.log-expected.json
@@ -7,7 +7,6 @@
         "destination.packets": 1,
         "destination.port": 53,
         "event.category": [
-            "network",
             "network"
         ],
         "event.dataset": "zeek.connection",
@@ -66,7 +65,6 @@
         "destination.packets": 1,
         "destination.port": 53,
         "event.category": [
-            "network",
             "network"
         ],
         "event.dataset": "zeek.connection",
@@ -124,7 +122,6 @@
         "destination.packets": 1,
         "destination.port": 53,
         "event.category": [
-            "network",
             "network"
         ],
         "event.dataset": "zeek.connection",
@@ -180,7 +177,6 @@
         "destination.ip": "198.51.100.249",
         "destination.packets": 0,
         "event.category": [
-            "network",
             "network"
         ],
         "event.dataset": "zeek.connection",

diff --git a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml
@@ -15,6 +15,7 @@ processors:
 - append:
     field: related.ip
     value: '{{source.ip}}'
+    if: ctx?.source?.ip != null
 - geoip:
     field: source.ip
     target_field: source.geo
@@ -37,6 +38,7 @@ processors:
 - append:
     field: related.ip
     value: '{{destination.ip}}'
+    if: ctx?.destination?.ip != null
 - geoip:
     field: destination.ip
     target_field: destination.geo

diff --git a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml
@@ -23,6 +23,14 @@ processors:
 - lowercase:
     field: event.action
     ignore_missing: true
+- append:
+    field: related.ip
+    value: '{{source.ip}}'
+    if: ctx?.source?.ip != null
+- append:
+    field: related.ip
+    value: '{{destination.ip}}'
+    if: ctx?.destination?.ip != null
 - geoip:
     field: destination.ip
     target_field: destination.geo

diff --git a/x-pack/filebeat/module/zeek/dnp3/test/dnp3-json.log-expected.json b/x-pack/filebeat/module/zeek/dnp3/test/dnp3-json.log-expected.json
@@ -23,6 +23,10 @@
         "network.community_id": "1:E57Z1w3RrSdR+fi6rSZblbQVhzY=",
         "network.protocol": "dnp3",
         "network.transport": "tcp",
+        "related.ip": [
+            "127.0.0.1",
+            "127.0.0.1"
+        ],
         "service.type": "zeek",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml
@@ -179,7 +179,6 @@ processors:
         - {from: source.address, to: source.ip, type: ip}
         - {from: destination.address, to: destination.ip, type: ip}
         - {from: zeek.session_id, to: event.id}
-        - {from: '@timestamp', to: event.created}
         - {from: zeek.dns.trans_id, to: dns.id}
         - {from: zeek.dns.query, to: dns.question.name}
         - {from: zeek.dns.qtype_name, to: dns.question.type}
@@ -197,18 +196,12 @@ processors:
 {{ if .community_id }}
   - community_id:
 {{ end }}
-  - timestamp:
-      ignore_missing: true
-      field: zeek.dns.ts
-      layouts:
-        - UNIX
   - drop_fields:
       ignore_missing: true
       fields:
         - zeek.dns.Z
         - zeek.dns.auth
         - zeek.dns.addl
-        - zeek.dns.ts
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml
@@ -5,6 +5,15 @@ processors:
   - set:
         field: event.ingested
         value: '{{_ingest.timestamp}}'
+  - set:
+      field: event.created
+      value: '{{@timestamp}}'
+  - date:
+      field: zeek.dns.ts
+      formats:
+        - UNIX
+  - remove:
+      field: zeek.dns.ts
 
   # IP Geolocation Lookup
   - geoip:

diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml
@@ -14,10 +14,6 @@ processors:
   - decode_json_fields:
       fields: [event.original]
       target: zeek.intel
-  - timestamp:
-      field: zeek.intel.ts
-      layouts: [UNIX]
-      test: "1573030980.989353"
   - convert:
       ignore_missing: true
       fields:
@@ -57,7 +53,6 @@ processors:
   - drop_fields:
       ignore_missing: true
       fields:
-        - zeek.intel.ts
         - zeek.intel.id.orig_h
         - zeek.intel.id.orig_p
         - zeek.intel.id.resp_h

diff --git a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml
@@ -7,7 +7,12 @@ processors:
   - set:
       field: event.created
       value: '{{@timestamp}}'
-
+  - date:
+      field: zeek.intel.ts
+      formats:
+        - UNIX
+  - remove:
+      field: zeek.intel.ts
   # IP Geolocation Lookup
   - geoip:
       if: ctx.source?.geo == null

diff --git a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml
@@ -71,11 +71,11 @@ processors:
     field: event.type
     value: error
     if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status != 'succeeded'"
-- append:
+- set:
     field: event.outcome
     value: success
     if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status == 'succeeded'"
-- append:
+- set:
     field: event.outcome
     value: failure
     if: "ctx?.zeek?.socks?.status != null && ctx.zeek.socks.status != 'succeeded'"

diff --git a/x-pack/filebeat/module/zeek/socks/test/socks-json.log-expected.json b/x-pack/filebeat/module/zeek/socks/test/socks-json.log-expected.json
@@ -11,9 +11,7 @@
         "event.id": "Cmz4Cb4qCw1hGqYw1c",
         "event.kind": "event",
         "event.module": "zeek",
-        "event.outcome": [
-            "success"
-        ],
+        "event.outcome": "success",
         "event.type": [
             "connection",
             "protocol"

diff --git a/x-pack/filebeat/module/zeek/ssl/config/ssl.yml b/x-pack/filebeat/module/zeek/ssl/config/ssl.yml
@@ -72,7 +72,7 @@ processors:
         kind: event
         category:
           - network
-        kind:
+        type:
           - connection
           - protocol
 {{ if .community_id }}

diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml
@@ -65,7 +65,7 @@ processors:
 - kv:
     field: zeek.ssl.issuer
     field_split: ','
-    value_split: =
+    value_split: '='
     target_field: zeek.ssl.server.issuer
     ignore_missing: true
 - rename:
@@ -104,7 +104,7 @@ processors:
 - kv:
     field: zeek.ssl.subject
     field_split: ','
-    value_split: =
+    value_split: '='
     target_field: zeek.ssl.server.subject
     ignore_missing: true
 - remove:
@@ -142,7 +142,7 @@ processors:
 - kv:
     field: zeek.ssl.client_issuer
     field_split: ','
-    value_split: =
+    value_split: '='
     target_field: zeek.ssl.client.issuer
     ignore_missing: true
 - rename:
@@ -181,7 +181,7 @@ processors:
 - kv:
     field: zeek.ssl.client_subject
     field_split: ','
-    value_split: =
+    value_split: '='
     target_field: zeek.ssl.client.subject
     ignore_missing: true
 - remove:

diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json
@@ -19,11 +19,12 @@
         ],
         "event.dataset": "zeek.ssl",
         "event.id": "CAOvs1BMFCX2Eh0Y3",
-        "event.kind": [
+        "event.kind": "event",
+        "event.module": "zeek",
+        "event.type": [
             "connection",
             "protocol"
         ],
-        "event.module": "zeek",
         "fileset.name": "ssl",
         "input.type": "log",
         "log.offset": 0,
@@ -90,11 +91,12 @@
         ],
         "event.dataset": "zeek.ssl",
         "event.id": "C3mki91FnnNtm0u1ok",
-        "event.kind": [
+        "event.kind": "event",
+        "event.module": "zeek",
+        "event.type": [
             "connection",
             "protocol"
         ],
-        "event.module": "zeek",
         "fileset.name": "ssl",
         "input.type": "log",
         "log.offset": 635,