Make event.original optional

elastic · Jun 8, 2021 · 6269436 · 6269436
1 parent c335d60
commit 6269436
Show file tree

Hide file tree

Showing 185 changed files with 2,951 additions and 1,654 deletions.
diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.7.3"
+  changes:
+    - description: Make event.original optional
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/992
 - version: "0.7.2"
   changes:
     - description: adding back 0.7.0 changes

diff --git a/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-config.yml b/packages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-config.yml
@@ -2,3 +2,5 @@ dynamic_fields:
   event.ingested: ".*"
 fields:
   "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event
diff --git a/...ages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json b/...ages/zeek/data_stream/capture_loss/_dev/test/pipeline/test-capture-loss.log-expected.json
@@ -15,11 +15,15 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333182200Z",
+                "ingested": "2021-06-08T07:47:55.330460300Z",
+                "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "@timestamp": "2021-03-30T00:04:00.941Z",
@@ -36,11 +40,15 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333229400Z",
+                "ingested": "2021-06-08T07:47:55.330466Z",
+                "original": "{\"ts\":1617062640.941952,\"ts_delta\":900.0005369186401,\"peer\":\"zeek\",\"gaps\":58475,\"acks\":65665,\"percent_lost\":89.05048351481003}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "@timestamp": "2021-03-30T00:19:00.942Z",
@@ -57,11 +65,15 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333237400Z",
+                "ingested": "2021-06-08T07:47:55.330473300Z",
+                "original": "{\"ts\":1617063540.942231,\"ts_delta\":900.0002789497376,\"peer\":\"zeek\",\"gaps\":54754,\"acks\":61818,\"percent_lost\":88.5729075673752}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "@timestamp": "2021-03-30T00:34:00.942Z",
@@ -78,11 +90,15 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333243200Z",
+                "ingested": "2021-06-08T07:47:55.330482600Z",
+                "original": "{\"ts\":1617064440.942597,\"ts_delta\":900.0003659725189,\"peer\":\"zeek\",\"gaps\":51022,\"acks\":57974,\"percent_lost\":88.00841756649533}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "@timestamp": "2021-03-30T00:49:00.942Z",
@@ -99,11 +115,15 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333248500Z",
+                "ingested": "2021-06-08T07:47:55.330489900Z",
+                "original": "{\"ts\":1617065340.942651,\"ts_delta\":900.0000541210175,\"peer\":\"zeek\",\"gaps\":55105,\"acks\":62497,\"percent_lost\":88.17223226714883}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         },
         {
             "@timestamp": "2019-09-10T16:19:28.465Z",
@@ -128,12 +148,15 @@
                 }
             },
             "event": {
-                "ingested": "2021-04-23T19:56:22.333253200Z",
+                "ingested": "2021-06-08T07:47:55.330494Z",
                 "original": "{\"ts\":1568132368.465338,\"ts_delta\":32.282249,\"peer\":\"bro\",\"gaps\":0,\"acks\":206,\"percent_lost\":0.0}",
                 "type": "info",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "metric"
-            }
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }
diff --git a/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs b/packages/zeek/data_stream/capture_loss/agent/stream/httpjson.yml.hbs
@@ -39,3 +39,6 @@ tags:
 {{#each tags as |tag i|}}
   - {{tag}}
 {{/each}}
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
diff --git a/packages/zeek/data_stream/capture_loss/agent/stream/log.yml.hbs b/packages/zeek/data_stream/capture_loss/agent/stream/log.yml.hbs
@@ -9,6 +9,9 @@ tags:
 {{#each tags as |tag i|}}
  - {{tag}}
 {{/each}}
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
 {{/contains}}
diff --git a/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/capture_loss/elasticsearch/ingest_pipeline/default.yml
@@ -11,44 +11,52 @@ processors:
   - set:
       field: ecs.version
       value: '1.9.0'
-  - json:
+  - rename:
       field: message
-      target_field: json
-      ignore_failure: true
-      if: ctx?.message != null
+      target_field: event.original
+  - json:
+      field: event.original
+      target_field: _temp_.json
   - drop:
-      if: 'ctx?.json?.result == null && ctx?.json?.ts == null' 
-  - rename:
-      field: json
-      target_field: zeek.capture_loss
-      if: ctx?.json?.ts != null
+      description: Drop if no Splunk or log data present.
+      if: 'ctx?._temp_?.json?.result == null && ctx?._temp_?.json?.ts == null' 
+# Splunk specific parsing start
   - fingerprint:
       fields: 
-        - json.result._cd
-        - json.result._indextime
-        - json.result._raw
-        - json.result._time
-        - json.result.host
-        - json.result.source
+        - _temp_.json.result._cd
+        - _temp_.json.result._indextime
+        - _temp_.json.result._raw
+        - _temp_.json.result._time
+        - _temp_.json.result.host
+        - _temp_.json.result.source
       target_field: '_id'
-      if: 'ctx?.json?.result != null && ctx?.json?.ts == null'
-  - rename:
-      field: json.result._raw
-      target_field: event.original
+      if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null'
+  - remove:
+      field: event.original
       ignore_missing: true
-  - json:
+      if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null'
+  - set:
       field: event.original
-      target_field: zeek.capture_loss
+      copy_from: _temp_.json.result._raw
+      ignore_empty_value: true
       ignore_failure: true
-      if: ctx?.event?.original != null
+      if: 'ctx?._temp_?.json?.result != null && ctx?._temp_?.json?.ts == null'
   - rename:
-      field: json.result.host
+      field: _temp_.json.result.host
       target_field: host.name
       ignore_missing: true
   - rename:
-      field: json.result.source
+      field: _temp_.json.result.source
       target_field: log.file.path
       ignore_missing: true
+  - remove:
+      field: _temp_
+      ignore_missing: true
+# Splunk parsing end
+  - json:
+      field: event.original
+      target_field: zeek.capture_loss
+      ignore_failure: true
   - date:
       field: zeek.capture_loss.ts
       formats:
@@ -69,6 +77,11 @@ processors:
         - message
         - json
       ignore_missing: true
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-config.yml b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-config.yml
@@ -2,3 +2,5 @@ dynamic_fields:
   event.ingested: ".*"
 fields:
   "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event