elastic · P1llus · Jun 9, 2021 · Jun 8, 2021 · Jun 8, 2021
diff --git a/packages/traefik/changelog.yml b/packages/traefik/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: update to ECS 1.10.0 and adding event.original options
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1107
 - version: "0.1.2"
   changes:
     - description: setting minimum Kibana version required to 7.13.0

diff --git a/...ipeline/test-format-common.log-config.yml → ..._dev/test/pipeline/test-common-config.yml b/...ipeline/test-format-common.log-config.yml → ..._dev/test/pipeline/test-common-config.yml
@@ -1,4 +1,6 @@
 fields:
   "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event
 dynamic_fields:
   event.ingested: ".*"
diff --git a/packages/traefik/data_stream/access/_dev/test/pipeline/test-format-common.log-expected.json b/packages/traefik/data_stream/access/_dev/test/pipeline/test-format-common.log-expected.json
@@ -16,10 +16,16 @@
             "url": {
                 "original": "/ui/favicons/favicon-16x16.png"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "network": {
                 "transport": "tcp"
             },
             "@timestamp": "2017-10-02T20:22:07.000Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "192.168.33.1"
@@ -40,7 +46,8 @@
             },
             "event": {
                 "duration": 2000000,
-                "ingested": "2021-04-23T12:57:45.746442788Z",
+                "ingested": "2021-06-08T12:48:40.206985600Z",
+                "original": "192.168.33.1 - - [02/Oct/2017:20:22:07 +0000] \"GET /ui/favicons/favicon-16x16.png HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 262 \"Host-host-1\" \"http://172.19.0.3:5601\" 2ms",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "category": [
@@ -100,10 +107,16 @@
             "url": {
                 "original": "/ui/favicons/favicon.ico"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "network": {
                 "transport": "tcp"
             },
             "@timestamp": "2017-10-02T20:22:08.000Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "85.181.35.98"
@@ -124,7 +137,8 @@
             },
             "event": {
                 "duration": 3000000,
-                "ingested": "2021-04-23T12:57:45.746445997Z",
+                "ingested": "2021-06-08T12:48:40.207010400Z",
+                "original": "85.181.35.98 - - [02/Oct/2017:20:22:08 +0000] \"GET /ui/favicons/favicon.ico HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 271 \"Host-host1\" \"http://172.19.0.3:5601\" 3ms",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "category": [
@@ -184,10 +198,16 @@
             "url": {
                 "original": "/en/"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "network": {
                 "transport": "tcp"
             },
             "@timestamp": "2018-02-28T17:30:33.000Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "70.29.80.15"
@@ -207,7 +227,8 @@
             },
             "event": {
                 "duration": 247000000,
-                "ingested": "2021-04-23T12:57:45.746447030Z",
+                "ingested": "2021-06-08T12:48:40.207017Z",
+                "original": "70.29.80.15 - - [28/Feb/2018:17:30:33 +0000] \"GET /en/ HTTP/2.0\" 200 2814 - \"Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1\" 13 \"Host-host1-com-0\" \"http://172.19.0.6:14008\" 247ms",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "category": [
@@ -251,10 +272,16 @@
             "url": {
                 "original": "/"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "network": {
                 "transport": "tcp"
             },
             "@timestamp": "2018-11-29T15:03:51.000Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "::1"
@@ -275,7 +302,8 @@
             },
             "event": {
                 "duration": 0,
-                "ingested": "2021-04-23T12:57:45.746447905Z",
+                "ingested": "2021-06-08T12:48:40.207023400Z",
+                "original": "::1 - - [29/Nov/2018:15:03:51 +0000] \"GET / HTTP/1.1\" 404 19 \"-\" \"curl/7.62.0\" 10 \"backend not found\" \"/\" 0ms",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "category": [
@@ -332,10 +360,16 @@
             "url": {
                 "original": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "network": {
                 "transport": "tcp"
             },
             "@timestamp": "2018-01-19T10:01:02.000Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "94.254.131.115"
@@ -355,7 +389,8 @@
             },
             "event": {
                 "duration": 13000000,
-                "ingested": "2021-04-23T12:57:45.746448775Z",
+                "ingested": "2021-06-08T12:48:40.207028400Z",
+                "original": "94.254.131.115 - - [19/Jan/2018:10:01:02 +0000] \"GET /assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo HTTP/1.1\" 200 85 - \"Android\" 623112 \"Host-api-wearerealitygames-com-2\" \"http://172.25.0.9:4140\" 13ms",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "category": [
@@ -414,10 +449,16 @@
             "url": {
                 "original": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "network": {
                 "transport": "tcp"
             },
             "@timestamp": "2018-01-19T10:01:02.000Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "89.64.35.193"
@@ -437,7 +478,8 @@
             },
             "event": {
                 "duration": 8000000,
-                "ingested": "2021-04-23T12:57:45.746449644Z",
+                "ingested": "2021-06-08T12:48:40.207033100Z",
+                "original": "89.64.35.193 - - [19/Jan/2018:10:01:02 +0000] \"GET /marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM HTTP/1.1\" 200 150 - \"Android\" 623114 \"Host-api-wearerealitygames-com-2\" \"http://172.25.0.6:4140\" 8ms",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "category": [
@@ -463,7 +505,28 @@
             }
         },
         {
+            "traefik": {
+                "access": {
+                    "user_identifier": "-"
+                }
+            },
+            "source": {
+                "address": "127.0.0.1",
+                "ip": "127.0.0.1"
+            },
+            "url": {
+                "original": "/apache_pb.gif"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "network": {
+                "transport": "tcp"
+            },
             "@timestamp": "2000-10-10T20:55:36.000Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "user": [
                     "frank"
@@ -472,11 +535,6 @@
                     "127.0.0.1"
                 ]
             },
-            "traefik": {
-                "access": {
-                    "user_identifier": "-"
-                }
-            },
             "http": {
                 "request": {
                     "method": "GET"
@@ -489,30 +547,21 @@
                     "status_code": 200
                 }
             },
-            "source": {
-                "address": "127.0.0.1",
-                "ip": "127.0.0.1"
-            },
             "event": {
-                "ingested": "2021-04-23T12:57:45.746450500Z",
+                "ingested": "2021-06-08T12:48:40.207037600Z",
+                "original": "127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326",
+                "created": "2020-04-28T11:07:58.223Z",
+                "kind": "event",
                 "category": [
                     "web"
                 ],
                 "type": [
                     "access"
                 ],
-                "created": "2020-04-28T11:07:58.223Z",
-                "kind": "event",
                 "outcome": "success"
             },
             "user": {
                 "name": "frank"
-            },
-            "url": {
-                "original": "/apache_pb.gif"
-            },
-            "network": {
-                "transport": "tcp"
             }
         }
     ]

diff --git a/packages/traefik/data_stream/access/_dev/test/pipeline/test-format-json.log-config.yml b/packages/traefik/data_stream/access/_dev/test/pipeline/test-format-json.log-config.yml
diff --git a/packages/traefik/data_stream/access/_dev/test/pipeline/test-format-json.log-expected.json b/packages/traefik/data_stream/access/_dev/test/pipeline/test-format-json.log-expected.json
@@ -17,10 +17,16 @@
                 "original": "/",
                 "domain": "backend.elastic-package-service.docker.localhost"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "network": {
                 "transport": "tcp"
             },
             "@timestamp": "2021-03-16T18:56:54Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "127.0.0.1"
@@ -40,7 +46,8 @@
             },
             "event": {
                 "duration": 40356,
-                "ingested": "2021-04-23T12:57:45.974895554Z",
+                "ingested": "2021-06-08T12:48:40.515007700Z",
+                "original": "{\"BackendAddr\":\"\",\"BackendName\":\"Traefik\",\"BackendURL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"ClientAddr\":\"127.0.0.1:48658\",\"ClientHost\":\"127.0.0.1\",\"ClientPort\":\"48658\",\"ClientUsername\":\"-\",\"DownstreamContentSize\":19,\"DownstreamStatus\":404,\"DownstreamStatusLine\":\"404 Not Found\",\"Duration\":40356,\"FrontendName\":\"backend not found\",\"OriginContentSize\":19,\"OriginDuration\":4086,\"OriginStatus\":404,\"OriginStatusLine\":\"404 Not Found\",\"Overhead\":36270,\"RequestAddr\":\"backend.elastic-package-service.docker.localhost\",\"RequestContentSize\":0,\"RequestCount\":7,\"RequestHost\":\"backend.elastic-package-service.docker.localhost\",\"RequestLine\":\"GET / HTTP/1.1\",\"RequestMethod\":\"GET\",\"RequestPath\":\"/\",\"RequestPort\":\"-\",\"RequestProtocol\":\"HTTP/1.1\",\"RetryAttempts\":0,\"StartLocal\":\"2021-03-16T18:56:54.735539596Z\",\"StartUTC\":\"2021-03-16T18:56:54.735539596Z\",\"downstream_Content-Type\":\"text/plain; charset=utf-8\",\"downstream_X-Content-Type-Options\":\"nosniff\",\"level\":\"info\",\"msg\":\"\",\"origin_Content-Type\":\"text/plain; charset=utf-8\",\"origin_X-Content-Type-Options\":\"nosniff\",\"request_Accept\":\"*/*\",\"request_User-Agent\":\"curl/7.67.0\",\"time\":\"2021-03-16T18:56:54Z\"}",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "category": [
@@ -85,11 +92,17 @@
                 "original": "/",
                 "domain": "backend.docker.docker.localhost"
             },
+            "tags": [
+                "preserve_original_event"
+            ],
             "network": {
                 "community_id": "1:DJlJOSbrvisPNQtgBIyBaYAwlz8=",
                 "transport": "tcp"
             },
             "@timestamp": "2021-03-16T19:08:41Z",
+            "ecs": {
+                "version": "1.10.0"
+            },
             "related": {
                 "ip": [
                     "172.21.0.1",
@@ -110,7 +123,8 @@
             },
             "event": {
                 "duration": 3034764,
-                "ingested": "2021-04-23T12:57:45.974899317Z",
+                "ingested": "2021-06-08T12:48:40.515028400Z",
+                "original": "{\"BackendAddr\":\"172.21.0.2:80\",\"BackendName\":\"backend-backend-docker\",\"BackendURL\":{\"Scheme\":\"http\",\"Opaque\":\"\",\"User\":null,\"Host\":\"172.21.0.2:80\",\"Path\":\"\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"ClientAddr\":\"172.21.0.1:59068\",\"ClientHost\":\"172.21.0.1\",\"ClientPort\":\"59068\",\"ClientUsername\":\"-\",\"DownstreamContentSize\":383,\"DownstreamStatus\":200,\"DownstreamStatusLine\":\"200 OK\",\"Duration\":3034764,\"FrontendName\":\"Host-backend-docker-docker-localhost-2\",\"OriginContentSize\":383,\"OriginDuration\":2155389,\"OriginStatus\":200,\"OriginStatusLine\":\"200 OK\",\"Overhead\":879375,\"RequestAddr\":\"backend.docker.docker.localhost\",\"RequestContentSize\":0,\"RequestCount\":27,\"RequestHost\":\"backend.docker.docker.localhost\",\"RequestLine\":\"GET / HTTP/1.1\",\"RequestMethod\":\"GET\",\"RequestPath\":\"/\",\"RequestPort\":\"-\",\"RequestProtocol\":\"HTTP/1.1\",\"RetryAttempts\":0,\"StartLocal\":\"2021-03-16T19:08:41.039598834Z\",\"StartUTC\":\"2021-03-16T19:08:41.039598834Z\",\"downstream_Content-Length\":\"383\",\"downstream_Content-Type\":\"text/plain; charset=utf-8\",\"downstream_Date\":\"Tue, 16 Mar 2021 19:08:41 GMT\",\"level\":\"info\",\"msg\":\"\",\"origin_Content-Length\":\"383\",\"origin_Content-Type\":\"text/plain; charset=utf-8\",\"origin_Date\":\"Tue, 16 Mar 2021 19:08:41 GMT\",\"request_Accept\":\"*/*\",\"request_User-Agent\":\"curl/7.64.1\",\"time\":\"2021-03-16T19:08:41Z\"}",
                 "created": "2020-04-28T11:07:58.223Z",
                 "kind": "event",
                 "category": [

diff --git a/packages/traefik/data_stream/access/agent/stream/log.yml.hbs b/packages/traefik/data_stream/access/agent/stream/log.yml.hbs
@@ -2,9 +2,18 @@ paths:
 {{#each paths as |path i|}}
  - {{path}}
 {{/each}}
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
 exclude_files: [".gz$"]
+{{#if processors}}
 processors:
-  - add_fields:
-      target: ''
-      fields:
-        ecs.version: 1.9.0
+{{processors}}
+{{/if}}
diff --git a/packages/traefik/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/traefik/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -4,11 +4,19 @@ processors:
   - set:
       field: event.ingested
       value: '{{_ingest.timestamp}}'
+  - set:
+      field: ecs.version
+      value: "1.10.0"
   - rename:
       field: '@timestamp'
       target_field: event.created
-  - grok:
+      ignore_missing: true
+  - rename:
       field: message
+      target_field: event.original
+      ignore_missing: true
+  - grok:
+      field: event.original
       patterns:
         - ^%{CHAR:first_char}
       pattern_definitions:
@@ -89,6 +97,11 @@ processors:
       value: tcp
   - community_id:
       if: "ctx?.source?.ip != null && ctx?.source?.port != null && ctx?.destination?.ip != null && ctx?.destination?.ip != null"
+  - remove:
+      field: event.original
+      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/traefik/data_stream/access/elasticsearch/ingest_pipeline/format-common.yml b/packages/traefik/data_stream/access/elasticsearch/ingest_pipeline/format-common.yml
@@ -3,7 +3,7 @@ description: Pipeline for parsing Traefik access logs in commonlog format. Requi
   plugins.
 processors:
 - dissect:
-    field: message
+    field: event.original
     pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}]
       "%{http.request.method} %{url.original} HTTP/%{http.version}" %{http.response.status_code}
       %{traefik.access.message}'
@@ -15,9 +15,6 @@ processors:
       (?:"%{DATA:traefik.access.frontend_name}"|-)?)?( "%{DATA:traefik.access.backend_url}")?(
       %{NUMBER:temp.duration:long}ms)?)?
     ignore_missing: true
-- remove:
-    field: message
-    ignore_missing: true
 - remove:
     field: traefik.access.message
     ignore_missing: true