From 971f0654e7e78d47da403b0a2739dd7bb627c015 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 29 Nov 2018 15:24:42 -0500 Subject: [PATCH 01/12] Perform straightforward ECS renames: - kibana.log.meta.req.headers.referer => http.request.referrer - kibana.log.meta.req.headers.user-agent => user_agent.original - kibana.log.meta.req.remoteAddress => source.ip - kibana.log.meta.req.url => url.original --- .../module/kibana/log/ingest/pipeline.json | 30 +++++++++++++++++++ .../kibana/log/test/test.log-expected.json | 10 +++---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index 2e4d42814a8..af642712603 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -74,6 +74,36 @@ "ignore_missing": true } }, + + { + "rename": { + "field": "kibana.log.meta.req.headers.referer", + "target_field": "http.request.referrer", + "ignore_missing": true + } + }, + { + "rename": { + "field": "kibana.log.meta.req.headers.user-agent", + "target_field": "user_agent.original", + "ignore_missing": true + } + }, + { + "rename": { + "field": "kibana.log.meta.req.remoteAddress", + "target_field": "source.ip", + "ignore_missing": true + } + }, + { + "rename": { + "field": "kibana.log.meta.req.url", + "target_field": "url.original", + "ignore_missing": true + } + }, + { "date": { "field": "read_timestamp", diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index fbc7301d87e..3971176cdc9 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -6,6 +6,7 @@ "event.module": "kibana", "fileset.name": "log", "http.request.method": "get", + "http.request.referrer": "http://localhost:5601/app/kibana", "http.response.content_length": 9, "http.response.elapsed_time": 26, "http.response.status_code": 304, @@ -19,11 +20,7 @@ "kibana.log.meta.req.headers.if-modified-since": "Thu, 03 May 2018 09:45:28 GMT", "kibana.log.meta.req.headers.if-none-match": "\"24234c1c81b3948758c1a0be8e5a65386ca94c52\"", "kibana.log.meta.req.headers.origin": "http://localhost:5601", - "kibana.log.meta.req.headers.referer": "http://localhost:5601/app/kibana", - "kibana.log.meta.req.headers.user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", "kibana.log.meta.req.referer": "http://localhost:5601/app/kibana", - "kibana.log.meta.req.remoteAddress": "127.0.0.1", - "kibana.log.meta.req.url": "/ui/fonts/open_sans/open_sans_v15_latin_600.woff2", "kibana.log.meta.req.userAgent": "127.0.0.1", "kibana.log.meta.statusCode": 304, "kibana.log.meta.type": "response", @@ -33,7 +30,10 @@ "process.pid": 69410, "service.name": [ "kibana" - ] + ], + "source.ip": "127.0.0.1", + "url.original": "/ui/fonts/open_sans/open_sans_v15_latin_600.woff2", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36" }, { "@timestamp": "2018-05-09T10:59:12.000Z", From 722c830a8a0e0e3b4d2d438d7138947666b5f3d3 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 29 Nov 2018 15:25:54 -0500 Subject: [PATCH 02/12] Remove fields that were straight duplicates (will still be aliased): - kibana.log.meta.req.referer - kibana.log.meta.statusCode - kibana.log.meta.method --- .../module/kibana/log/ingest/pipeline.json | 19 +++++++++++++++++++ .../kibana/log/test/test.log-expected.json | 3 --- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index af642712603..b6ec60a040a 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -104,6 +104,25 @@ } }, + { + "remove": { + "field": "kibana.log.meta.req.referer", + "ignore_missing": true + } + }, + { + "remove": { + "field": "kibana.log.meta.statusCode", + "ignore_missing": true + } + }, + { + "remove": { + "field": "kibana.log.meta.method", + "ignore_missing": true + } + }, + { "date": { "field": "read_timestamp", diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index 3971176cdc9..7abb7ea8745 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -11,7 +11,6 @@ "http.response.elapsed_time": 26, "http.response.status_code": 304, "input.type": "log", - "kibana.log.meta.method": "get", "kibana.log.meta.req.headers.accept": "*/*", "kibana.log.meta.req.headers.accept-encoding": "gzip, deflate, br", "kibana.log.meta.req.headers.accept-language": "en-US,en;q=0.9,de;q=0.8", @@ -20,9 +19,7 @@ "kibana.log.meta.req.headers.if-modified-since": "Thu, 03 May 2018 09:45:28 GMT", "kibana.log.meta.req.headers.if-none-match": "\"24234c1c81b3948758c1a0be8e5a65386ca94c52\"", "kibana.log.meta.req.headers.origin": "http://localhost:5601", - "kibana.log.meta.req.referer": "http://localhost:5601/app/kibana", "kibana.log.meta.req.userAgent": "127.0.0.1", - "kibana.log.meta.statusCode": 304, "kibana.log.meta.type": "response", "kibana.log.tags": [], "log.offset": 0, From d66e68168f9f2e726e8bd0cd1eb2e180044e44b0 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 29 Nov 2018 15:34:11 -0500 Subject: [PATCH 03/12] Uppercase `http.request.method` field --- filebeat/module/kibana/log/ingest/pipeline.json | 6 ++++++ .../module/kibana/log/test/test.log-expected.json | 11 ++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index b6ec60a040a..1386b469b8d 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -74,6 +74,12 @@ "ignore_missing": true } }, + { + "uppercase": { + "field": "http.request.method", + "ignore_missing": true + } + }, { "rename": { diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index 7abb7ea8745..5d194c7c4bf 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -2,10 +2,9 @@ { "@timestamp": "2018-05-09T10:57:55.000Z", "ecs.version": "1.0.0-beta2", - "event.dataset": "kibana.log", + "event.dataset": "log", "event.module": "kibana", - "fileset.name": "log", - "http.request.method": "get", + "http.request.method": "GET", "http.request.referrer": "http://localhost:5601/app/kibana", "http.response.content_length": 9, "http.response.elapsed_time": 26, @@ -35,9 +34,8 @@ { "@timestamp": "2018-05-09T10:59:12.000Z", "ecs.version": "1.0.0-beta2", - "event.dataset": "kibana.log", + "event.dataset": "log", "event.module": "kibana", - "fileset.name": "log", "input.type": "log", "kibana.log.meta.type": "log", "kibana.log.tags": [ @@ -55,9 +53,8 @@ { "@timestamp": "2018-05-09T10:59:12.000Z", "ecs.version": "1.0.0-beta2", - "event.dataset": "kibana.log", + "event.dataset": "log", "event.module": "kibana", - "fileset.name": "log", "input.type": "log", "kibana.log.meta.type": "log", "kibana.log.tags": [ From 89a171f4482e5f9f1a9e26aebfbc78ccfc234700 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 30 Nov 2018 11:04:25 -0500 Subject: [PATCH 04/12] Compute `event.duration` --- filebeat/module/kibana/log/ingest/pipeline.json | 8 ++++++++ filebeat/module/kibana/log/test/test.log-expected.json | 1 + 2 files changed, 9 insertions(+) diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index 1386b469b8d..9d3a6b1d0bc 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -53,6 +53,14 @@ "ignore_missing": true } }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * params.scale)", + "params": { "scale": 1000000 }, + "if": "ctx.kibana.log.containsKey('meta') && ctx.kibana.log.meta.containsKey('res') && ctx.kibana.log.meta.res.containsKey('responseTime')" + } + }, { "rename": { "field": "kibana.log.meta.res.responseTime", diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index 5d194c7c4bf..a991ffaa6af 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -3,6 +3,7 @@ "@timestamp": "2018-05-09T10:57:55.000Z", "ecs.version": "1.0.0-beta2", "event.dataset": "log", + "event.duration": 26000000, "event.module": "kibana", "http.request.method": "GET", "http.request.referrer": "http://localhost:5601/app/kibana", From a3f15c4216e8da8080cdd7119ae642b83ab2dc29 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 16:17:22 -0500 Subject: [PATCH 05/12] Alias transitioned fields --- filebeat/docs/fields.asciidoc | 63 +++++++++++++++++++ filebeat/module/kibana/fields.go | 2 +- filebeat/module/kibana/log/_meta/fields.yml | 29 +++++++++ .../module/kibana/log/ingest/pipeline.json | 13 ++-- .../kibana/log/test/test.log-expected.json | 2 +- 5 files changed, 101 insertions(+), 8 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4dec70fe19b..3d82d72f5b9 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -5720,6 +5720,69 @@ type: object -- +*`kibana.log.kibana.log.meta.req.headers.referer`*:: ++ +-- +type: alias + +alias to: http.request.referrer + +-- + +*`kibana.log.kibana.log.meta.req.referer`*:: ++ +-- +type: alias + +alias to: http.request.referrer + +-- + +*`kibana.log.kibana.log.meta.req.headers.user-agent`*:: ++ +-- +type: alias + +alias to: user_agent.original + +-- + +*`kibana.log.kibana.log.meta.req.remoteAddress`*:: ++ +-- +type: alias + +alias to: source.address + +-- + +*`kibana.log.kibana.log.meta.req.url`*:: ++ +-- +type: alias + +alias to: url.original + +-- + +*`kibana.log.kibana.log.meta.meta.statusCode`*:: ++ +-- +type: alias + +alias to: http.response.status_code + +-- + +*`kibana.log.kibana.log.meta.method`*:: ++ +-- +type: alias + +alias to: http.request.method + +-- + [[exported-fields-kubernetes-processor]] == Kubernetes fields diff --git a/filebeat/module/kibana/fields.go b/filebeat/module/kibana/fields.go index 3ef7bd487d8..ad600fd669c 100644 --- a/filebeat/module/kibana/fields.go +++ b/filebeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJyskEFuwyAQRfc+xVf2yQFYdNNl1TNUkzBGFMJYMFbl21cxboUt2lVm+T9674szAi8GwV8p0QCo18gGpxqcBsByuWU/qZdk8DIA2F7jXewceQBGz9EWs3ZnJLpzQ3ycLhMbuCzztCUd6p7TsqK436wH+xNY743GQA8Iok9cLk15NLZWJVd2xY868PIl2R66fwasI+qfRXHOJ7fCL11tUVJ+nvd1zpmTVixk3Ib03XdW6qrl+sk3PVQ1/NiP+w4AAP//OeSYZw==" + return "eJzMlEHOmzAQhfecYvTv4wOwqFRlWfUM0QQPxsXYdDxWldtXYFIRx2naKIvfCxYz+HuPZw8HGOnSwmjP6LEBECuOWvjIhY8GQFPs2M5ig2/hSwMA29vwPejkqAHoLTkd27V3AI8T7YjLkstMLRgOad4qFeotZ89ywfyp1WAPgXl9w37EBQLOeopq1ywV96qCJt40rtIjXX4F1kXvLwZWEzkzF4yx3qxwVZWNgkLv0z0mZvKSsRD6zUhdeyLBqnQ4/6BOilYunurmqvx8KZQLRi1SiumnGgg1cVRMPTFxVR6dxfIsZpShhUFkXjCJomTEPWOyhjHnI5zon519PkfXrFIkPqAhXx7JM3PLxtO6UQW2xnp0bwtrCkJftWaK9bF57CqGxB0prG5+1VDi8tOehsPunamsj2XsUjwGXR/pp9cozsFH2jCn7p7zorMhlP+R/7rXVUJp5XcAAAD//31usoE=" } diff --git a/filebeat/module/kibana/log/_meta/fields.yml b/filebeat/module/kibana/log/_meta/fields.yml index f7f87416490..10dbb093b34 100644 --- a/filebeat/module/kibana/log/_meta/fields.yml +++ b/filebeat/module/kibana/log/_meta/fields.yml @@ -14,3 +14,32 @@ - name: meta type: object object_type: keyword + + - name: kibana.log.meta.req.headers.referer + type: alias + path: http.request.referrer + migration: true + - name: kibana.log.meta.req.referer + type: alias + path: http.request.referrer + migration: true + - name: kibana.log.meta.req.headers.user-agent + type: alias + path: user_agent.original + migration: true + - name: kibana.log.meta.req.remoteAddress + type: alias + path: source.address + migration: true + - name: kibana.log.meta.req.url + type: alias + path: url.original + migration: true + - name: kibana.log.meta.meta.statusCode + type: alias + path: http.response.status_code + migration: true + - name: kibana.log.meta.method + type: alias + path: http.request.method + migration: true diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index 9d3a6b1d0bc..e2bf6446475 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -82,12 +82,6 @@ "ignore_missing": true } }, - { - "uppercase": { - "field": "http.request.method", - "ignore_missing": true - } - }, { "rename": { @@ -106,6 +100,13 @@ { "rename": { "field": "kibana.log.meta.req.remoteAddress", + "target_field": "source.address", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.address", "target_field": "source.ip", "ignore_missing": true } diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index a991ffaa6af..f052cf05c77 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -5,7 +5,7 @@ "event.dataset": "log", "event.duration": 26000000, "event.module": "kibana", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://localhost:5601/app/kibana", "http.response.content_length": 9, "http.response.elapsed_time": 26, From 5edc9ebc584fedbc5cf0665a8df939fc29452fe4 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 16:17:48 -0500 Subject: [PATCH 06/12] Document transitioned fields in ecs-migration.yml --- dev-tools/ecs-migration.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 06c6f39630e..199251c5ec4 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -404,6 +404,36 @@ to: message alias: true +## Kibana module + +- from: kibana.log.meta.req.headers.referer + to: http.request.referrer + alias: true + +- from: kibana.log.meta.req.referer + to: http.request.referrer + alias: true + +- from: kibana.log.meta.req.headers.user-agent + to: user_agent.original + alias: true + +- from: kibana.log.meta.req.remoteAddress + to: source.address + alias: true + +- from: kibana.log.meta.req.url + to: url.original + alias: true + +- from: kibana.log.meta.meta.statusCode + to: http.response.status_code + alias: true + +- from: kibana.log.meta.method + to: http.request.method + alias: true + ## NGINX module - from: nginx.access.user_name From 50cb96b8b971166358c50dd125f48a713a3d924e Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 16:32:05 -0500 Subject: [PATCH 07/12] Fix a typo in one of the transitioned fields --- filebeat/docs/fields.asciidoc | 2 +- filebeat/module/kibana/fields.go | 2 +- filebeat/module/kibana/log/_meta/fields.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 3d82d72f5b9..28a18b8bcc2 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -5765,7 +5765,7 @@ alias to: url.original -- -*`kibana.log.kibana.log.meta.meta.statusCode`*:: +*`kibana.log.kibana.log.meta.statusCode`*:: + -- type: alias diff --git a/filebeat/module/kibana/fields.go b/filebeat/module/kibana/fields.go index ad600fd669c..59fdee6b6ac 100644 --- a/filebeat/module/kibana/fields.go +++ b/filebeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzMlEHOmzAQhfecYvTv4wOwqFRlWfUM0QQPxsXYdDxWldtXYFIRx2naKIvfCxYz+HuPZw8HGOnSwmjP6LEBECuOWvjIhY8GQFPs2M5ig2/hSwMA29vwPejkqAHoLTkd27V3AI8T7YjLkstMLRgOad4qFeotZ89ywfyp1WAPgXl9w37EBQLOeopq1ywV96qCJt40rtIjXX4F1kXvLwZWEzkzF4yx3qxwVZWNgkLv0z0mZvKSsRD6zUhdeyLBqnQ4/6BOilYunurmqvx8KZQLRi1SiumnGgg1cVRMPTFxVR6dxfIsZpShhUFkXjCJomTEPWOyhjHnI5zon519PkfXrFIkPqAhXx7JM3PLxtO6UQW2xnp0bwtrCkJftWaK9bF57CqGxB0prG5+1VDi8tOehsPunamsj2XsUjwGXR/pp9cozsFH2jCn7p7zorMhlP+R/7rXVUJp5XcAAAD//31usoE=" + return "eJzMlEHO2yAQhfc+xejfhwN4UanKsuoZookZY2oM7jCoyu0rG6dyCGnaKIuf5Uz43stjxgcY6dLCaM/osQEQK45a+MiFjwZAU+zYzmKDb+FLAwDbr+F70MlRA9Bbcjq2a+8AHifaEZcjl5laMBzSvFUq1FvOnuWC+VOrwR4C8/mG/YgLBJz1FNWuWSruVQVNvGlcpUe6/Aqsi95fDKwmcmYuGGO9WeGqKhsFhd6ne0zM5CVjIfSbkbr2RIJV6XD+QZ0UrVw81c1V+XkolAtGLVKK6acaCDVxVEw9MXFVHp3F8i1mlKGFQWReMImiZMQ9Y7KGMecjnOifnX0+R9esUiQ+oCFfPskzc8vF03pRBbbGenRvC2sKQl+1Zor1tXnsKobEHSmsXn7VUOLyrz0Nh907U1k2LsVj0PVtfjpBcQ4+0oY5dfecV0xNJEMoPyH/NdJVQmnldwAAAP//jzOwrA==" } diff --git a/filebeat/module/kibana/log/_meta/fields.yml b/filebeat/module/kibana/log/_meta/fields.yml index 10dbb093b34..9ef1c657806 100644 --- a/filebeat/module/kibana/log/_meta/fields.yml +++ b/filebeat/module/kibana/log/_meta/fields.yml @@ -35,7 +35,7 @@ type: alias path: url.original migration: true - - name: kibana.log.meta.meta.statusCode + - name: kibana.log.meta.statusCode type: alias path: http.response.status_code migration: true From 2b58151040d82bf4ba3f3b50cb0e879c5f2bef0b Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 16:34:17 -0500 Subject: [PATCH 08/12] Changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a385332ed54..95b623f5e7e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Modify apache/error dataset to follow ECS. {pull}8963[8963] - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] +- Rename many `kibana.log.*` fields to map to ECS. {pull}9301[9301] *Heartbeat* From 4f76e40f8710d27a339cb3b063d1df07948eacb4 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 3 Jan 2019 15:51:56 -0500 Subject: [PATCH 09/12] Don't rename source.address, but copy it to source.ip --- .../module/kibana/log/ingest/pipeline.json | 7 ++-- .../kibana/log/test/test.log-expected.json | 39 +------------------ 2 files changed, 4 insertions(+), 42 deletions(-) diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index e2bf6446475..a4a55117c16 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -105,10 +105,9 @@ } }, { - "rename": { - "field": "source.address", - "target_field": "source.ip", - "ignore_missing": true + "set": { + "field": "source.ip", + "value": "{{source.address}}" } }, { diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index f052cf05c77..dd32a8ddd54 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -28,46 +28,9 @@ "service.name": [ "kibana" ], + "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/ui/fonts/open_sans/open_sans_v15_latin_600.woff2", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36" - }, - { - "@timestamp": "2018-05-09T10:59:12.000Z", - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "kibana", - "input.type": "log", - "kibana.log.meta.type": "log", - "kibana.log.tags": [ - "debug", - "monitoring-ui", - "kibana-monitoring" - ], - "log.offset": 920, - "message": "Fetching data from kibana_stats collector", - "process.pid": 69776, - "service.name": [ - "kibana" - ] - }, - { - "@timestamp": "2018-05-09T10:59:12.000Z", - "ecs.version": "1.0.0-beta2", - "event.dataset": "log", - "event.module": "kibana", - "input.type": "log", - "kibana.log.meta.type": "log", - "kibana.log.tags": [ - "reporting", - "debug", - "exportTypes" - ], - "log.offset": 1090, - "message": "Found exportType at /Users/ruflin/Downloads/6.3/kibana-6.3.0-darwin-x86_64/node_modules/x-pack/plugins/reporting/export_types/csv/server/index.js", - "process.pid": 69776, - "service.name": [ - "kibana" - ] } ] \ No newline at end of file From 9a8aa8a6888dbe6946f6154b7d1b02040acdb5c2 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 4 Jan 2019 10:26:20 -0500 Subject: [PATCH 10/12] Only copy `source.address` to `source.ip` if address is present --- .../module/kibana/log/ingest/pipeline.json | 3 +- .../kibana/log/test/test.log-expected.json | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index a4a55117c16..5ede04d9dc8 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -107,7 +107,8 @@ { "set": { "field": "source.ip", - "value": "{{source.address}}" + "value": "{{source.address}}", + "if": "ctx.containsKey('source') && ctx.source.containsKey('address')" } }, { diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index dd32a8ddd54..59ba0ad8b7b 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -32,5 +32,43 @@ "source.ip": "127.0.0.1", "url.original": "/ui/fonts/open_sans/open_sans_v15_latin_600.woff2", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36" + }, + { + "@timestamp": "2018-05-09T10:59:12.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "kibana", + "input.type": "log", + "kibana.log.meta.type": "log", + "kibana.log.tags": [ + "debug", + "monitoring-ui", + "kibana-monitoring" + ], + "log.offset": 920, + "message": "Fetching data from kibana_stats collector", + "process.pid": 69776, + "service.name": [ + "kibana" + ] + }, + { + "@timestamp": "2018-05-09T10:59:12.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "log", + "event.module": "kibana", + "input.type": "log", + "kibana.log.meta.type": "log", + "kibana.log.tags": [ + "reporting", + "debug", + "exportTypes" + ], + "log.offset": 1090, + "message": "Found exportType at /Users/ruflin/Downloads/6.3/kibana-6.3.0-darwin-x86_64/node_modules/x-pack/plugins/reporting/export_types/csv/server/index.js", + "process.pid": 69776, + "service.name": [ + "kibana" + ] } ] \ No newline at end of file From b7592a6d8213abdf75338686b3089fadee3c84f0 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 8 Jan 2019 16:03:28 -0500 Subject: [PATCH 11/12] Regenerate expected file. Was missing the recent `event.dataset` fix --- filebeat/module/kibana/log/test/test.log-expected.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/filebeat/module/kibana/log/test/test.log-expected.json b/filebeat/module/kibana/log/test/test.log-expected.json index 59ba0ad8b7b..77283f2539b 100644 --- a/filebeat/module/kibana/log/test/test.log-expected.json +++ b/filebeat/module/kibana/log/test/test.log-expected.json @@ -2,9 +2,10 @@ { "@timestamp": "2018-05-09T10:57:55.000Z", "ecs.version": "1.0.0-beta2", - "event.dataset": "log", + "event.dataset": "kibana.log", "event.duration": 26000000, "event.module": "kibana", + "fileset.name": "log", "http.request.method": "get", "http.request.referrer": "http://localhost:5601/app/kibana", "http.response.content_length": 9, @@ -36,8 +37,9 @@ { "@timestamp": "2018-05-09T10:59:12.000Z", "ecs.version": "1.0.0-beta2", - "event.dataset": "log", + "event.dataset": "kibana.log", "event.module": "kibana", + "fileset.name": "log", "input.type": "log", "kibana.log.meta.type": "log", "kibana.log.tags": [ @@ -55,8 +57,9 @@ { "@timestamp": "2018-05-09T10:59:12.000Z", "ecs.version": "1.0.0-beta2", - "event.dataset": "log", + "event.dataset": "kibana.log", "event.module": "kibana", + "fileset.name": "log", "input.type": "log", "kibana.log.meta.type": "log", "kibana.log.tags": [ From dfd5670ec2996188996d365aca9e7131d78d00bb Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 10 Jan 2019 13:46:00 -0500 Subject: [PATCH 12/12] Use the very nice null-safe operators to shorten `if` clauses --- filebeat/module/kibana/log/ingest/pipeline.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/module/kibana/log/ingest/pipeline.json b/filebeat/module/kibana/log/ingest/pipeline.json index 5ede04d9dc8..0822a0624c4 100755 --- a/filebeat/module/kibana/log/ingest/pipeline.json +++ b/filebeat/module/kibana/log/ingest/pipeline.json @@ -58,7 +58,7 @@ "lang": "painless", "source": "ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * params.scale)", "params": { "scale": 1000000 }, - "if": "ctx.kibana.log.containsKey('meta') && ctx.kibana.log.meta.containsKey('res') && ctx.kibana.log.meta.res.containsKey('responseTime')" + "if": "ctx.kibana.log.meta?.res?.responseTime != null" } }, { @@ -108,7 +108,7 @@ "set": { "field": "source.ip", "value": "{{source.address}}", - "if": "ctx.containsKey('source') && ctx.source.containsKey('address')" + "if": "ctx.source?.address != null" } }, {