From 3c13de562d9806adf5246e640c449bff0d184e5a Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Thu, 12 Mar 2020 14:50:32 -0500 Subject: [PATCH] [Filebeat] Improve ECS categorization in elasticsearch module (#16469) - event.kind - event.category - event.type - event.outcome - lowercase http.request.method - host.id - host.name - related.user Closes #16160 --- CHANGELOG.next.asciidoc | 1 + .../audit/ingest/pipeline-json.json | 229 ------------- .../audit/ingest/pipeline-json.yml | 124 ++++++++ .../audit/ingest/pipeline-plaintext.json | 87 ----- .../audit/ingest/pipeline-plaintext.yml | 62 ++++ .../elasticsearch/audit/ingest/pipeline.json | 54 ---- .../elasticsearch/audit/ingest/pipeline.yml | 66 ++++ .../module/elasticsearch/audit/manifest.yml | 6 +- .../audit/test/test-access.log-expected.json | 60 ++++ .../test/test-audit-711.log-expected.json | 21 ++ .../test/test-audit-730.log-expected.json | 70 ++++ .../test/test-audit-docker.log-expected.json | 15 +- .../audit/test/test-audit.log-expected.json | 51 ++- .../deprecation/ingest/pipeline-json.json | 112 ------- .../deprecation/ingest/pipeline-json.yml | 56 ++++ .../ingest/pipeline-plaintext.json | 47 --- .../deprecation/ingest/pipeline-plaintext.yml | 37 +++ .../deprecation/ingest/pipeline.json | 54 ---- .../deprecation/ingest/pipeline.yml | 43 +++ .../elasticsearch/deprecation/manifest.yml | 6 +- ...lasticsearch_deprecation.log-expected.json | 12 + ...lasticsearch_deprecation.log-expected.json | 48 +++ .../test/test-json.log-expected.json | 52 +++ .../elasticsearch/gc/ingest/pipeline.json | 59 ---- .../elasticsearch/gc/ingest/pipeline.yml | 62 ++++ filebeat/module/elasticsearch/gc/manifest.yml | 2 +- .../gc/test/gc.log-expected.json | 300 ++++++++++++++++++ .../gc/test/test.log-expected.json | 9 + .../server/ingest/pipeline-json.json | 122 ------- .../server/ingest/pipeline-json.yml | 68 ++++ .../server/ingest/pipeline-plaintext.json | 53 ---- .../server/ingest/pipeline-plaintext.yml | 43 +++ .../elasticsearch/server/ingest/pipeline.json | 78 ----- .../elasticsearch/server/ingest/pipeline.yml | 90 ++++++ .../module/elasticsearch/server/manifest.yml | 6 +- .../test/elasticsearch.624.log-expected.json | 177 +++++++++++ .../server/test/test-json.log-expected.json | 184 +++++++++++ .../server/test/test.log-expected.json | 57 ++++ .../slowlog/ingest/pipeline-json.json | 140 -------- .../slowlog/ingest/pipeline-json.yml | 76 +++++ .../slowlog/ingest/pipeline-plaintext.json | 56 ---- .../slowlog/ingest/pipeline-plaintext.yml | 41 +++ .../slowlog/ingest/pipeline.json | 70 ---- .../elasticsearch/slowlog/ingest/pipeline.yml | 60 ++++ .../module/elasticsearch/slowlog/manifest.yml | 6 +- ...g_index_indexing_slowlog.log-expected.json | 18 ++ ...ex_indexing_slowlog-json.log-expected.json | 8 + ...ndex_search_slowlog-json.log-expected.json | 36 +++ ...ex_indexing_slowlog-json.log-expected.json | 8 + ...ndex_search_slowlog-json.log-expected.json | 12 + .../test/slowlogs-json.log-expected.json | 20 ++ .../slowlog/test/test.log-expected.json | 21 ++ 52 files changed, 2018 insertions(+), 1177 deletions(-) delete mode 100644 filebeat/module/elasticsearch/audit/ingest/pipeline-json.json create mode 100644 filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml delete mode 100644 filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json create mode 100644 filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.yml delete mode 100644 filebeat/module/elasticsearch/audit/ingest/pipeline.json create mode 100644 filebeat/module/elasticsearch/audit/ingest/pipeline.yml delete mode 100755 filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json create mode 100644 filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml delete mode 100755 filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json create mode 100644 filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml delete mode 100644 filebeat/module/elasticsearch/deprecation/ingest/pipeline.json create mode 100644 filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml delete mode 100644 filebeat/module/elasticsearch/gc/ingest/pipeline.json create mode 100644 filebeat/module/elasticsearch/gc/ingest/pipeline.yml delete mode 100644 filebeat/module/elasticsearch/server/ingest/pipeline-json.json create mode 100644 filebeat/module/elasticsearch/server/ingest/pipeline-json.yml delete mode 100755 filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json create mode 100644 filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.yml delete mode 100644 filebeat/module/elasticsearch/server/ingest/pipeline.json create mode 100644 filebeat/module/elasticsearch/server/ingest/pipeline.yml delete mode 100644 filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json create mode 100644 filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml delete mode 100644 filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json create mode 100644 filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.yml delete mode 100644 filebeat/module/elasticsearch/slowlog/ingest/pipeline.json create mode 100644 filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7e9b1756f6d..45b3e5979d6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -177,6 +177,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354] - Improve ECS categorization field mapping in icinga module. {issue}16164[16164] {pull}16533[16533] - Improve ECS categorization field mappings in ibmmq module. {issue}16163[16163] {pull}16532[16532] +- Improve ECS categorization, host field mappings in elasticsearch module. {issue}16160[16160] {pull}16469[16469] *Heartbeat* diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json deleted file mode 100644 index 1a1dcdcb794..00000000000 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json +++ /dev/null @@ -1,229 +0,0 @@ -{ - "description": "Pipeline for parsing elasticsearch audit logs in JSON format", - "processors": [ - { - "json": { - "field": "message", - "target_field": "elasticsearch.audit" - } - }, - { - "drop": { - "if": "ctx.elasticsearch.audit?.type != null && ctx.elasticsearch.audit.type != 'audit'" - } - }, - { - "remove": { - "field": "elasticsearch.audit.type", - "ignore_missing": true - } - }, - { - "date": { - "if": "ctx.elasticsearch.audit['@timestamp'] != null && ctx.event.timezone != null", - "field": "elasticsearch.audit.@timestamp", - "target_field": "elasticsearch.audit.@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "timezone": "{{ event.timezone }}" - } - }, - { - "remove": { - "if": "ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null", - "field": "event.timezone" - } - }, - { - "rename": { - "field": "elasticsearch.audit.timestamp", - "target_field": "elasticsearch.audit.@timestamp", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "event.action", - "path": "elasticsearch.audit" - } - }, - { - "rename": { - "field": "elasticsearch.audit.event.action", - "target_field": "event.action", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "event.type", - "path": "elasticsearch.audit" - } - }, - { - "rename": { - "field": "elasticsearch.audit.event.type", - "target_field": "elasticsearch.audit.layer", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "origin.address", - "path": "elasticsearch.audit" - } - }, - { - "grok": { - "field": "elasticsearch.audit.origin.address", - "patterns": [ - "\\[%{IPORHOST:source.ip}\\]:%{INT:source.port:int}", - "%{IPORHOST:source.ip}:%{INT:source.port:int}" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "elasticsearch.audit.origin.address", - "target_field": "source.address", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "url.path", - "path": "elasticsearch.audit" - } - }, - { - "dot_expander": { - "field": "url.query", - "path": "elasticsearch.audit" - } - }, - { - "set": { - "if": "ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query == null", - "field": "url.original", - "value": "{{elasticsearch.audit.url.path}}" - } - }, - { - "set": { - "if": "ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query != null", - "field": "url.original", - "value": "{{elasticsearch.audit.url.path}}?{{elasticsearch.audit.url.query}}" - } - }, - { - "remove": { - "if": "ctx.elasticsearch.audit?.url?.path != null", - "field": "elasticsearch.audit.url.path" - } - }, - { - "remove": { - "if": "ctx.elasticsearch.audit?.url?.query != null", - "field": "elasticsearch.audit.url.query" - } - }, - { - "dot_expander": { - "field": "node.id", - "path": "elasticsearch.audit" - } - }, - { - "dot_expander": { - "field": "node.name", - "path": "elasticsearch.audit" - } - }, - { - "rename": { - "field": "elasticsearch.audit.node", - "target_field": "elasticsearch.node" - } - }, - { - "dot_expander": { - "field": "user.name", - "path": "elasticsearch.audit" - } - }, - { - "rename": { - "field": "elasticsearch.audit.user.name", - "target_field": "user.name", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "request.method", - "path": "elasticsearch.audit" - } - }, - { - "rename": { - "field": "elasticsearch.audit.request.method", - "target_field": "http.request.method", - "ignore_missing": true - } - - }, - { - "dot_expander": { - "field": "request.body", - "path": "elasticsearch.audit" - } - }, - { - "rename": { - "field": "elasticsearch.audit.request.body", - "target_field": "http.request.body.content", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "cluster.name", - "path": "elasticsearch.audit" - } - }, - { - "rename": { - "field": "elasticsearch.audit.cluster.name", - "target_field": "elasticsearch.cluster.name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "elasticsearch.audit.level", - "target_field": "log.level", - "ignore_missing": true - } - }, - { - "date": { - "field": "elasticsearch.audit.@timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - "ignore_failure": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml new file mode 100644 index 00000000000..434db5cab21 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml @@ -0,0 +1,124 @@ +description: Pipeline for parsing elasticsearch audit logs in JSON format +processors: +- json: + field: message + target_field: elasticsearch.audit +- drop: + if: ctx.elasticsearch.audit?.type != null && ctx.elasticsearch.audit.type != 'audit' +- remove: + field: elasticsearch.audit.type + ignore_missing: true +- date: + if: ctx.elasticsearch.audit['@timestamp'] != null && ctx.event.timezone != null + field: elasticsearch.audit.@timestamp + target_field: elasticsearch.audit.@timestamp + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + timezone: '{{ event.timezone }}' +- remove: + if: ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null + field: event.timezone +- rename: + field: elasticsearch.audit.timestamp + target_field: elasticsearch.audit.@timestamp + ignore_missing: true +- dot_expander: + field: event.action + path: elasticsearch.audit +- rename: + field: elasticsearch.audit.event.action + target_field: event.action + ignore_missing: true +- dot_expander: + field: event.type + path: elasticsearch.audit +- rename: + field: elasticsearch.audit.event.type + target_field: elasticsearch.audit.layer + ignore_missing: true +- dot_expander: + field: origin.address + path: elasticsearch.audit +- grok: + field: elasticsearch.audit.origin.address + patterns: + - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int} + - '%{IPORHOST:source.ip}:%{INT:source.port:int}' + ignore_missing: true +- rename: + field: elasticsearch.audit.origin.address + target_field: source.address + ignore_missing: true +- dot_expander: + field: url.path + path: elasticsearch.audit +- dot_expander: + field: url.query + path: elasticsearch.audit +- set: + if: ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query + == null + field: url.original + value: '{{elasticsearch.audit.url.path}}' +- set: + if: ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query + != null + field: url.original + value: '{{elasticsearch.audit.url.path}}?{{elasticsearch.audit.url.query}}' +- remove: + if: ctx.elasticsearch.audit?.url?.path != null + field: elasticsearch.audit.url.path +- remove: + if: ctx.elasticsearch.audit?.url?.query != null + field: elasticsearch.audit.url.query +- dot_expander: + field: node.id + path: elasticsearch.audit +- dot_expander: + field: node.name + path: elasticsearch.audit +- rename: + field: elasticsearch.audit.node + target_field: elasticsearch.node +- dot_expander: + field: user.name + path: elasticsearch.audit +- rename: + field: elasticsearch.audit.user.name + target_field: user.name + ignore_missing: true +- dot_expander: + field: request.method + path: elasticsearch.audit +- rename: + field: elasticsearch.audit.request.method + target_field: http.request.method + ignore_missing: true +- dot_expander: + field: request.body + path: elasticsearch.audit +- rename: + field: elasticsearch.audit.request.body + target_field: http.request.body.content + ignore_missing: true +- dot_expander: + field: cluster.name + path: elasticsearch.audit +- rename: + field: elasticsearch.audit.cluster.name + target_field: elasticsearch.cluster.name + ignore_missing: true +- rename: + field: elasticsearch.audit.level + target_field: log.level + ignore_missing: true +- date: + field: elasticsearch.audit.@timestamp + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json deleted file mode 100644 index 345df18be13..00000000000 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "description": "Pipeline for parsing elasticsearch audit logs in plaintext format", - "processors": [ - { - "grok": { - "field": "message", - "pattern_definitions": { - "ES_TIMESTAMP": "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.@timestamp}\\]", - "ES_NODE_NAME": "(\\[%{DATA:elasticsearch.node.name}\\])?", - "ES_AUDIT_LAYER": "\\[%{WORD:elasticsearch.audit.layer}\\]", - "ES_AUDIT_EVENT_TYPE": "\\[%{WORD:elasticsearch.audit.event_type}\\]", - "ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin.type}\\])?", - "ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:source.ip}\\])?", - "ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{DATA:user.name}\\])?", - "ES_AUDIT_REALM": "(realm\\=\\[%{WORD:elasticsearch.audit.realm}\\])?", - "ES_AUDIT_ROLES": "(roles\\=\\[%{DATA:elasticsearch.audit.user.roles}\\])?", - "ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}(\\[%{DATA:elasticsearch.audit.sub_action}\\])?\\])?", - "ES_AUDIT_URI": "(uri=\\[%{DATA:url.original}\\])?", - "ES_AUDIT_URI_PARAMS": "(params=\\[%{DATA:elasticsearch.audit.url.params}\\])?", - "ES_AUDIT_INDICES": "(indices\\=\\[%{DATA:elasticsearch.audit.indices}\\])?", - "ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request.name}\\])?", - "ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:http.request.body.content}\\])?" - }, - "patterns": [ - "%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_REALM},?\\s*%{ES_AUDIT_ROLES},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_INDICES},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_URI_PARAMS},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?" - ] - } - }, - { - "split": { - "field": "elasticsearch.audit.user.roles", - "separator": ",", - "ignore_missing": true - } - }, - { - "split": { - "field": "elasticsearch.audit.indices", - "separator": ",", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.elasticsearch.audit.sub_action != null) { ctx.elasticsearch.audit.action += '[' + ctx.elasticsearch.audit.sub_action + ']' }" - } - }, - { - "remove": { - "field": "elasticsearch.audit.sub_action", - "ignore_missing": true - } - }, - { - "date": { - "if": "ctx.event.timezone == null", - "field": "elasticsearch.audit.@timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "elasticsearch.audit.@timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.yml new file mode 100644 index 00000000000..29c4348124c --- /dev/null +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.yml @@ -0,0 +1,62 @@ +description: Pipeline for parsing elasticsearch audit logs in plaintext format +processors: +- grok: + field: message + pattern_definitions: + ES_TIMESTAMP: \[%{TIMESTAMP_ISO8601:elasticsearch.audit.@timestamp}\] + ES_NODE_NAME: (\[%{DATA:elasticsearch.node.name}\])? + ES_AUDIT_LAYER: \[%{WORD:elasticsearch.audit.layer}\] + ES_AUDIT_EVENT_TYPE: \[%{WORD:elasticsearch.audit.event_type}\] + ES_AUDIT_ORIGIN_TYPE: (origin_type\=\[%{WORD:elasticsearch.audit.origin.type}\])? + ES_AUDIT_ORIGIN_ADDRESS: (origin_address\=\[%{IPORHOST:source.ip}\])? + ES_AUDIT_PRINCIPAL: (principal\=\[%{DATA:user.name}\])? + ES_AUDIT_REALM: (realm\=\[%{WORD:elasticsearch.audit.realm}\])? + ES_AUDIT_ROLES: (roles\=\[%{DATA:elasticsearch.audit.user.roles}\])? + ES_AUDIT_ACTION: (action\=\[%{DATA:elasticsearch.audit.action}(\[%{DATA:elasticsearch.audit.sub_action}\])?\])? + ES_AUDIT_URI: (uri=\[%{DATA:url.original}\])? + ES_AUDIT_URI_PARAMS: (params=\[%{DATA:elasticsearch.audit.url.params}\])? + ES_AUDIT_INDICES: (indices\=\[%{DATA:elasticsearch.audit.indices}\])? + ES_AUDIT_REQUEST: (request\=\[%{WORD:elasticsearch.audit.request.name}\])? + ES_AUDIT_REQUEST_BODY: (request_body\=\[%{DATA:http.request.body.content}\])? + patterns: + - '%{ES_TIMESTAMP}\s*%{ES_NODE_NAME}\s*%{ES_AUDIT_LAYER}\s*%{ES_AUDIT_EVENT_TYPE}\s*%{ES_AUDIT_ORIGIN_TYPE},?\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\s*%{ES_AUDIT_PRINCIPAL},?\s*%{ES_AUDIT_REALM},?\s*%{ES_AUDIT_ROLES},?\s*%{ES_AUDIT_ACTION},?\s*%{ES_AUDIT_INDICES},?\s*%{ES_AUDIT_URI},?\s*%{ES_AUDIT_URI_PARAMS},?\s*%{ES_AUDIT_REQUEST},?\s*%{ES_AUDIT_REQUEST_BODY},?' +- split: + field: elasticsearch.audit.user.roles + separator: ',' + ignore_missing: true +- split: + field: elasticsearch.audit.indices + separator: ',' + ignore_missing: true +- script: + lang: painless + source: if (ctx.elasticsearch.audit.sub_action != null) { ctx.elasticsearch.audit.action + += '[' + ctx.elasticsearch.audit.sub_action + ']' } +- remove: + field: elasticsearch.audit.sub_action + ignore_missing: true +- date: + if: ctx.event.timezone == null + field: elasticsearch.audit.@timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx.event.timezone != null + field: elasticsearch.audit.@timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json deleted file mode 100644 index 282abdac7be..00000000000 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "description": "Pipeline for parsing elasticsearch audit logs", - "processors": [ - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "grok": { - "field": "message", - "patterns": [ - "^%{CHAR:first_char}" - ], - "pattern_definitions": { - "CHAR": "." - } - } - }, - { - "pipeline": { - "if": "ctx.first_char != '{'", - "name": "{< IngestPipeline "pipeline-plaintext" >}" - } - }, - { - "pipeline": { - "if": "ctx.first_char == '{'", - "name": "{< IngestPipeline "pipeline-json" >}" - } - }, - { - "remove": { - "field": "elasticsearch.audit.@timestamp" - } - }, - { - "remove": { - "field": [ - "first_char" - ] - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml new file mode 100644 index 00000000000..8ad600ca792 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -0,0 +1,66 @@ +description: Pipeline for parsing elasticsearch audit logs +processors: +- rename: + field: '@timestamp' + target_field: event.created +- grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . +- pipeline: + if: ctx.first_char != '{' + name: '{< IngestPipeline "pipeline-plaintext" >}' +- pipeline: + if: ctx.first_char == '{' + name: '{< IngestPipeline "pipeline-json" >}' +- set: + field: event.kind + value: event +- set: + field: event.category + value: database +- set: + if: "ctx?.elasticsearch?.audit?.event_type != null" + field: event.type + value: access +- script: + lang: painless + source: >- + def successEvents = ['authentication_success', 'access_granted', 'run_as_granted', 'connection_granted']; + if (ctx?.elasticsearch?.audit?.event_type != null && successEvents.contains(ctx.elasticsearch.audit.event_type)) { + ctx.event.outcome = 'success'; + } else { + ctx.event.outcome = 'failure'; + } + if (ctx?.event.action != null && successEvents.contains(ctx.event.action)) { + ctx.event.outcome = 'success'; + } else { + ctx.event.outcome = 'failure'; + } + +- lowercase: + field: http.request.method + ignore_missing: true +- set: + field: host.id + value: "{{elasticsearch.node.id}}" + if: "ctx?.elasticsearch?.node?.id != null" +- set: + field: host.name + value: "{{elasticsearch.node.name}}" + if: "ctx?.elasticsearch?.node?.name != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +- remove: + field: elasticsearch.audit.@timestamp +- remove: + field: + - first_char +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/audit/manifest.yml b/filebeat/module/elasticsearch/audit/manifest.yml index 1335a5c25c5..ad5116eff08 100644 --- a/filebeat/module/elasticsearch/audit/manifest.yml +++ b/filebeat/module/elasticsearch/audit/manifest.yml @@ -16,8 +16,8 @@ var: - c:/ProgramData/Elastic/Elasticsearch/logs/*_audit.json ingest_pipeline: - - ingest/pipeline.json - - ingest/pipeline-json.json - - ingest/pipeline-plaintext.json + - ingest/pipeline.yml + - ingest/pipeline-json.yml + - ingest/pipeline-plaintext.yml input: config/audit.yml diff --git a/filebeat/module/elasticsearch/audit/test/test-access.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-access.log-expected.json index 63674428530..79843144d65 100644 --- a/filebeat/module/elasticsearch/audit/test/test-access.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-access.log-expected.json @@ -3,13 +3,20 @@ "@timestamp": "2018-06-19T05:16:15.549-02:00", "elasticsearch.audit.event_type": "authentication_failed", "elasticsearch.audit.layer": "rest", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "input.type": "log", "log.offset": 0, "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", + "related.user": [ + "i030648" + ], "service.type": "elasticsearch", "source.ip": "147.107.128.77", "url.original": "/_xpack/security/_authenticate", @@ -20,13 +27,20 @@ "elasticsearch.audit.event_type": "authentication_failed", "elasticsearch.audit.layer": "rest", "elasticsearch.node.name": "v_VJhjV", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "input.type": "log", "log.offset": 155, "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", + "related.user": [ + "rado" + ], "service.type": "elasticsearch", "source.ip": "172.22.0.3", "url.original": "/_xpack/security/_authenticate", @@ -39,13 +53,20 @@ "elasticsearch.audit.layer": "transport", "elasticsearch.audit.origin.type": "local_node", "elasticsearch.audit.request.name": "ClearScrollRequest", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "input.type": "log", "log.offset": 306, "message": "[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", + "related.user": [ + "_xpack_security" + ], "service.type": "elasticsearch", "source.ip": "192.168.1.165", "user.name": "_xpack_security" @@ -55,9 +76,13 @@ "elasticsearch.audit.event_type": "anonymous_access_denied", "elasticsearch.audit.layer": "rest", "elasticsearch.node.name": "v_VJhjV", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "input.type": "log", "log.offset": 519, @@ -70,13 +95,20 @@ "@timestamp": "2018-06-19T05:26:27.268-02:00", "elasticsearch.audit.event_type": "authentication_failed", "elasticsearch.audit.layer": "rest", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "input.type": "log", "log.offset": 654, "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", + "related.user": [ + "N078801" + ], "service.type": "elasticsearch", "source.ip": "147.107.128.77", "url.original": "/_xpack/security/_authenticate", @@ -89,13 +121,20 @@ "elasticsearch.audit.layer": "transport", "elasticsearch.audit.origin.type": "rest", "elasticsearch.audit.request.name": "MainRequest", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "input.type": "log", "log.offset": 802, "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", + "related.user": [ + "_anonymous" + ], "service.type": "elasticsearch", "source.ip": "147.107.128.77", "user.name": "_anonymous" @@ -105,14 +144,21 @@ "elasticsearch.audit.event_type": "authentication_failed", "elasticsearch.audit.layer": "rest", "elasticsearch.node.name": "v_VJhjV", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "http.request.body.content": "body", "input.type": "log", "log.offset": 986, "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", + "related.user": [ + "elastic" + ], "service.type": "elasticsearch", "source.ip": "172.18.0.3", "url.original": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", @@ -140,13 +186,20 @@ "foo_reader" ], "elasticsearch.node.name": "NodeName-0", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "input.type": "log", "log.offset": 1210, "message": "[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted] origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]", + "related.user": [ + "username" + ], "service.type": "elasticsearch", "source.ip": "192.168.2.1", "user.name": "username" @@ -158,14 +211,21 @@ "elasticsearch.audit.realm": "default_file", "elasticsearch.audit.url.params": "{username=jacknich2}", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": "access", "fileset.name": "audit", "http.request.body.content": "{\"metadata\":{\"intelligence\":7},\"full_name\":\"Jack Nicholson\",\"roles\":[\"admin\",\"other_role1\"", "input.type": "log", "log.offset": 1626, "message": "[2019-01-27T20:04:27,244] [node-0] [rest] [authentication_success] origin_address=[::1], principal=[elastic-admin], realm=[default_file], uri=[/_xpack/security/user/jacknich2], params=[{username=jacknich2}], request_body=[{\"metadata\":{\"intelligence\":7},\"full_name\":\"Jack Nicholson\",\"roles\":[\"admin\",\"other_role1\"],\"email\":\"jacknich@example.com\"}]", + "related.user": [ + "elastic-admin" + ], "service.type": "elasticsearch", "source.ip": "::1", "url.original": "/_xpack/security/user/jacknich2", diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json index 01cf1d9b0d9..161d7542b04 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit-711.log-expected.json @@ -9,13 +9,20 @@ "elasticsearch.audit.request.name": "IndicesStatsRequest", "elasticsearch.node.id": "UwRu4mReRtyJO1-FWAPvIQ", "event.action": "authentication_success", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "UwRu4mReRtyJO1-FWAPvIQ", "input.type": "log", "log.offset": 0, "message": "{\"@timestamp\":\"2019-09-05T14:02:37,921\", \"node.id\":\"UwRu4mReRtyJO1-FWAPvIQ\", \"event.type\":\"transport\", \"event.action\":\"authentication_success\", \"user.name\":\"_system\", \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"realm\":\"__fallback\", \"request.id\":\"474ZciqtQteOhjLO3OdZIw\", \"action\":\"indices:monitor/stats\", \"request.name\":\"IndicesStatsRequest\"}", + "related.user": [ + "_system" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:9300", "source.ip": "127.0.0.1", @@ -35,13 +42,20 @@ ], "elasticsearch.node.id": "DJKjhISiTzy-JY5nCU8h3Q", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "DJKjhISiTzy-JY5nCU8h3Q", "input.type": "log", "log.offset": 363, "message": "{\"@timestamp\":\"2020-01-29T09:41:10,856\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear\", \"request.name\":\"ClearRealmCacheRequest\"}", + "related.user": [ + "_xpack_security" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:9300", "source.ip": "127.0.0.1", @@ -61,13 +75,20 @@ ], "elasticsearch.node.id": "DJKjhISiTzy-JY5nCU8h3Q", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "DJKjhISiTzy-JY5nCU8h3Q", "input.type": "log", "log.offset": 785, "message": "{\"@timestamp\":\"2020-01-29T09:41:10,859\", \"node.id\":\"DJKjhISiTzy-JY5nCU8h3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"request.id\":\"I9bQCw28Qfe4HWtIJHgoAg\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear[n]\", \"request.name\":\"Node\"}", + "related.user": [ + "_xpack_security" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:9300", "source.ip": "127.0.0.1", diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json index 5c1518d7ecd..0ee154a4023 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json @@ -16,12 +16,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 0, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:08,484-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53568\", \"request.id\":\"qxwx-kV9Q9uSwmaeQp1OgQ\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\"*\",\"-*\"]}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53568", "source.ip": "127.0.0.1", @@ -45,12 +52,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 423, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:08,484-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53566\", \"request.id\":\"YhpBByKQTvSJfqCmMyWtXg\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\"*\",\"-*\"]}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53566", "source.ip": "127.0.0.1", @@ -74,12 +88,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 846, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,084-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53562\", \"request.id\":\"h50kZy1fTM6F3uP7MyFPDw\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\"*\",\"-*\"]}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53562", "source.ip": "127.0.0.1", @@ -102,12 +123,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 1269, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,611-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53570\", \"request.id\":\"7KZfVjrYToq8LGLW5tcyDA\", \"action\":\"indices:data/read/search\", \"request.name\":\"SearchRequest\", \"indices\":[\".kibana_task_manager\"]}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53570", "source.ip": "127.0.0.1", @@ -130,12 +158,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 1706, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,612-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53570\", \"request.id\":\"7KZfVjrYToq8LGLW5tcyDA\", \"action\":\"indices:data/read/search[phase/query]\", \"request.name\":\"ShardSearchTransportRequest\", \"indices\":[\".kibana_task_manager\"]}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53570", "source.ip": "127.0.0.1", @@ -155,12 +190,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 2170, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,758-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53572\", \"request.id\":\"TAklb9dXRtSg5V9fybe2Kw\", \"action\":\"cluster:monitor/nodes/info\", \"request.name\":\"NodesInfoRequest\"}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53572", "source.ip": "127.0.0.1", @@ -180,12 +222,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 2576, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:09,758-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53572\", \"request.id\":\"TAklb9dXRtSg5V9fybe2Kw\", \"action\":\"cluster:monitor/nodes/info[n]\", \"request.name\":\"NodeInfoRequest\"}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53572", "source.ip": "127.0.0.1", @@ -208,12 +257,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 2984, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:11,366-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53571\", \"request.id\":\"7mgKUfdjQhmlrw7ZDG1FjQ\", \"action\":\"indices:data/read/get\", \"request.name\":\"GetRequest\", \"indices\":[\".kibana\"]}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53571", "source.ip": "127.0.0.1", @@ -236,12 +292,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 3402, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:11,372-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53571\", \"request.id\":\"7mgKUfdjQhmlrw7ZDG1FjQ\", \"action\":\"indices:data/read/get[s]\", \"request.name\":\"GetRequest\", \"indices\":[\".kibana\"]}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53571", "source.ip": "127.0.0.1", @@ -261,12 +324,19 @@ ], "elasticsearch.node.id": "MA2xjPZLSvmif8VZ86OJZw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "fileset.name": "audit", + "host.id": "MA2xjPZLSvmif8VZ86OJZw", "input.type": "log", "log.offset": 3823, "message": "{\"type\":\"audit\", \"timestamp\":\"2019-06-11T05:21:11,381-0700\", \"node.id\":\"MA2xjPZLSvmif8VZ86OJZw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"kibana\", \"user.realm\":\"reserved\", \"user.roles\":[\"kibana_system\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53574\", \"request.id\":\"7VcDBbunQnqT2_cGxtURXA\", \"action\":\"cluster:admin/xpack/monitoring/bulk\", \"request.name\":\"MonitoringBulkRequest\"}", + "related.user": [ + "kibana" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:53574", "source.ip": "127.0.0.1", diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json index a2da63f62fd..457f930622d 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json @@ -6,10 +6,14 @@ "elasticsearch.audit.request.id": "pkduyMB5Tly6xgmkYbZi-A", "elasticsearch.node.id": "Xaq2BFVcQ1OhyMrjL8gNOg", "event.action": "anonymous_access_denied", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "fileset.name": "audit", - "http.request.method": "GET", + "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg", + "http.request.method": "get", "input.type": "log", "log.offset": 0, "message": "{\"type\": \"audit\", \"timestamp\":\"2019-06-11T15:03:32,102+0000\", \"node.id\":\"Xaq2BFVcQ1OhyMrjL8gNOg\", \"event.type\":\"rest\", \"event.action\":\"anonymous_access_denied\", \"origin.type\":\"rest\", \"origin.address\":\"172.17.0.1:40380\", \"url.path\":\"/\", \"request.method\":\"GET\", \"request.id\":\"pkduyMB5Tly6xgmkYbZi-A\"}", @@ -26,13 +30,20 @@ "elasticsearch.audit.request.id": "KPgEINaXSbGNaIobp8OcMw", "elasticsearch.node.id": "Xaq2BFVcQ1OhyMrjL8gNOg", "event.action": "authentication_failed", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "fileset.name": "audit", - "http.request.method": "GET", + "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg", + "http.request.method": "get", "input.type": "log", "log.offset": 690, "message": "{\"type\": \"audit\", \"timestamp\":\"2019-06-11T15:03:32,778+0000\", \"node.id\":\"Xaq2BFVcQ1OhyMrjL8gNOg\", \"event.type\":\"rest\", \"event.action\":\"authentication_failed\", \"user.name\":\"elastic\", \"origin.type\":\"rest\", \"origin.address\":\"172.17.0.1:40380\", \"url.path\":\"/\", \"request.method\":\"GET\", \"request.id\":\"KPgEINaXSbGNaIobp8OcMw\"}", + "related.user": [ + "elastic" + ], "service.type": "elasticsearch", "source.address": "172.17.0.1:40380", "source.ip": "172.17.0.1", diff --git a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json index 07667974cd5..4d618682910 100644 --- a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json @@ -5,13 +5,20 @@ "elasticsearch.audit.origin.type": "rest", "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw", "event.action": "authentication_failed", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "DSiWcTyeThWtUXLB9J0BMw", "input.type": "log", "log.offset": 0, "message": "{\"@timestamp\":\"2018-10-31T09:34:25,109\", \"node.id\":\"DSiWcTyeThWtUXLB9J0BMw\", \"event.type\":\"rest\", \"event.action\":\"authentication_failed\", \"user.name\":\"elastic\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:61598\", \"url.path\":\"/_xpack/security/user/beats_system/_password\"}", + "related.user": [ + "elastic" + ], "service.type": "elasticsearch", "source.address": "[::1]:61598", "source.ip": "::1", @@ -25,13 +32,20 @@ "elasticsearch.audit.origin.type": "rest", "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw", "event.action": "authentication_failed", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "failure", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "DSiWcTyeThWtUXLB9J0BMw", "input.type": "log", "log.offset": 274, "message": "{\"@timestamp\":\"2018-10-31T09:34:25,207\", \"node.id\":\"DSiWcTyeThWtUXLB9J0BMw\", \"event.type\":\"rest\", \"event.action\":\"authentication_failed\", \"user.name\":\"elastic\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:61599\", \"url.path\":\"/_xpack/security/user/remote_monitoring_user/_password\"}", + "related.user": [ + "elastic" + ], "service.type": "elasticsearch", "source.address": "[::1]:61599", "source.ip": "::1", @@ -51,13 +65,20 @@ ], "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "DSiWcTyeThWtUXLB9J0BMw", "input.type": "log", "log.offset": 558, "message": "{\"@timestamp\":\"2018-10-31T09:35:11,428\", \"node.id\":\"DSiWcTyeThWtUXLB9J0BMw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear\", \"request.name\":\"ClearRealmCacheRequest\"}", + "related.user": [ + "_xpack_security" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:9300", "source.ip": "127.0.0.1", @@ -76,13 +97,20 @@ ], "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "DSiWcTyeThWtUXLB9J0BMw", "input.type": "log", "log.offset": 941, "message": "{\"@timestamp\":\"2018-10-31T09:35:11,430\", \"node.id\":\"DSiWcTyeThWtUXLB9J0BMw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"action\":\"cluster:admin/xpack/security/realm/cache/clear[n]\", \"request.name\":\"Node\"}", + "related.user": [ + "_xpack_security" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:9300", "source.ip": "127.0.0.1", @@ -101,13 +129,20 @@ ], "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "DSiWcTyeThWtUXLB9J0BMw", "input.type": "log", "log.offset": 1309, "message": "{\"@timestamp\":\"2018-10-31T09:35:12,303\", \"node.id\":\"DSiWcTyeThWtUXLB9J0BMw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\",\"origin.address\":\"[::1]:61711\", \"action\":\"cluster:admin/xpack/security/user/change_password\", \"request.name\":\"ChangePasswordRequest\"}", + "related.user": [ + "elastic" + ], "service.type": "elasticsearch", "source.address": "[::1]:61711", "source.ip": "::1", @@ -129,13 +164,20 @@ ], "elasticsearch.node.id": "DSiWcTyeThWtUXLB9J0BMw", "event.action": "access_granted", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "DSiWcTyeThWtUXLB9J0BMw", "input.type": "log", "log.offset": 1676, "message": "{\"@timestamp\":\"2018-10-31T09:35:12,314\", \"node.id\":\"DSiWcTyeThWtUXLB9J0BMw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"_xpack_security\", \"user.realm\":\"__attach\", \"user.roles\":[\"superuser\"], \"origin.type\":\"local_node\", \"origin.address\":\"127.0.0.1:9300\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\".security-6\"]}", + "related.user": [ + "_xpack_security" + ], "service.type": "elasticsearch", "source.address": "127.0.0.1:9300", "source.ip": "127.0.0.1", @@ -151,15 +193,22 @@ "elasticsearch.node.id": "y8fa3M5zSSGo1M_KJRMUXw", "elasticsearch.node.name": "node-0", "event.action": "authentication_success", + "event.category": "database", "event.dataset": "elasticsearch.audit", + "event.kind": "event", "event.module": "elasticsearch", + "event.outcome": "success", "event.timezone": "-02:00", "fileset.name": "audit", + "host.id": "y8fa3M5zSSGo1M_KJRMUXw", "http.request.body.content": "\n{\n \"query\" : {\n \"term\" : { \"user\" : \"kimchy\" }\n }\n}\n", - "http.request.method": "GET", + "http.request.method": "get", "input.type": "log", "log.offset": 2056, "message": "{\"@timestamp\":\"2019-01-27T20:15:10,380\", \"node.name\":\"node-0\", \"node.id\":\"y8fa3M5zSSGo1M_KJRMUXw\", \"event.type\":\"rest\", \"event.action\":\"authentication_success\", \"user.name\":\"elastic-admin\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:58955\", \"realm\":\"default_file\", \"url.path\":\"/_search\", \"request.method\":\"GET\", \"request.body\":\"\\n{\\n \\\"query\\\" : {\\n \\\"term\\\" : { \\\"user\\\" : \\\"kimchy\\\" }\\n }\\n}\\n\", \"request.id\":\"WzL_kb6VSvOhAq0twPvHOQ\"}", + "related.user": [ + "elastic-admin" + ], "service.type": "elasticsearch", "source.address": "[::1]:58955", "source.ip": "::1", diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json deleted file mode 100755 index 00fe1b14f85..00000000000 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "description": "Pipeline for parsing the Elasticsearch deprecation log file in JSON format.", - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ], - "processors": [ - { - "json": { - "field": "message", - "target_field": "elasticsearch.deprecation" - } - }, - { - "drop": { - "if": "ctx.elasticsearch.deprecation.type != 'deprecation'" - } - }, - { - "remove": { - "field": "elasticsearch.deprecation.type" - } - }, - { - "rename": { - "field": "elasticsearch.deprecation.level", - "target_field": "log.level" - } - }, - { - "rename": { - "field": "elasticsearch.deprecation.component", - "target_field": "elasticsearch.component" - } - }, - { - "dot_expander": { - "field": "cluster.name", - "path": "elasticsearch.deprecation" - } - }, - { - "rename": { - "field": "elasticsearch.deprecation.cluster.name", - "target_field": "elasticsearch.cluster.name" - } - }, - { - "dot_expander": { - "field": "node.name", - "path": "elasticsearch.deprecation" - } - }, - { - "rename": { - "field": "elasticsearch.deprecation.node.name", - "target_field": "elasticsearch.node.name" - } - }, - { - "dot_expander": { - "field": "cluster.uuid", - "path": "elasticsearch.deprecation" - } - }, - { - "rename": { - "field": "elasticsearch.deprecation.cluster.uuid", - "target_field": "elasticsearch.cluster.uuid", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "node.id", - "path": "elasticsearch.deprecation" - } - }, - { - "rename": { - "field": "elasticsearch.deprecation.node.id", - "target_field": "elasticsearch.node.id", - "ignore_missing": true - } - }, - { - "remove": { - "field": "message" - } - }, - { - "rename": { - "field": "elasticsearch.deprecation.message", - "target_field": "message" - } - }, - { - "date": { - "field": "elasticsearch.deprecation.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - "ignore_failure": true - } - } - ] -} diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml new file mode 100644 index 00000000000..43c9bbdd6e2 --- /dev/null +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml @@ -0,0 +1,56 @@ +description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + target_field: elasticsearch.deprecation +- drop: + if: ctx.elasticsearch.deprecation.type != 'deprecation' +- remove: + field: elasticsearch.deprecation.type +- rename: + field: elasticsearch.deprecation.level + target_field: log.level +- rename: + field: elasticsearch.deprecation.component + target_field: elasticsearch.component +- dot_expander: + field: cluster.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- remove: + field: message +- rename: + field: elasticsearch.deprecation.message + target_field: message +- date: + field: elasticsearch.deprecation.timestamp + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json deleted file mode 100755 index c4276664547..00000000000 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for parsing the Elasticsearch deprecation log file in plaintext format.", - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ], - "processors": [ - { - "grok": { - "field": "message", - "pattern_definitions": { - "GREEDYMULTILINE": "(.|\n)*" - }, - "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}\\]\\[%{DATA:elasticsearch.component}%{SPACE}\\] %{GREEDYMULTILINE:message}" - ] - } - }, - { - "date": { - "if": "ctx.event.timezone == null", - "field": "elasticsearch.deprecation.timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "elasticsearch.deprecation.timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - } - ] -} diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml new file mode 100644 index 00000000000..433d6ba53dc --- /dev/null +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml @@ -0,0 +1,37 @@ +description: Pipeline for parsing the Elasticsearch deprecation log file in plaintext + format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- grok: + field: message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + patterns: + - \[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\]\[%{LOGLEVEL:log.level}%{SPACE}\]\[%{DATA:elasticsearch.component}%{SPACE}\] + %{GREEDYMULTILINE:message} +- date: + if: ctx.event.timezone == null + field: elasticsearch.deprecation.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx.event.timezone != null + field: elasticsearch.deprecation.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json deleted file mode 100644 index 59bf8e42868..00000000000 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "description": "Pipeline for parsing elasticsearch deprecation logs", - "processors": [ - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "grok": { - "field": "message", - "patterns": [ - "^%{CHAR:first_char}" - ], - "pattern_definitions": { - "CHAR": "." - } - } - }, - { - "pipeline": { - "if": "ctx.first_char != '{'", - "name": "{< IngestPipeline "pipeline-plaintext" >}" - } - }, - { - "pipeline": { - "if": "ctx.first_char == '{'", - "name": "{< IngestPipeline "pipeline-json" >}" - } - }, - { - "remove": { - "field": "elasticsearch.deprecation.timestamp" - } - }, - { - "remove": { - "field": [ - "first_char" - ] - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml new file mode 100644 index 00000000000..1fab99c0b16 --- /dev/null +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml @@ -0,0 +1,43 @@ +description: Pipeline for parsing elasticsearch deprecation logs +processors: +- rename: + field: '@timestamp' + target_field: event.created +- grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . +- pipeline: + if: ctx.first_char != '{' + name: '{< IngestPipeline "pipeline-plaintext" >}' +- pipeline: + if: ctx.first_char == '{' + name: '{< IngestPipeline "pipeline-json" >}' +- set: + field: event.kind + value: event +- set: + field: event.category + value: database +- set: + field: event.type + value: info +- set: + field: host.id + value: "{{elasticsearch.node.id}}" + if: "ctx?.elasticsearch?.node?.id != null" +- set: + field: host.name + value: "{{elasticsearch.node.name}}" + if: "ctx?.elasticsearch?.node?.name != null" +- remove: + field: elasticsearch.deprecation.timestamp +- remove: + field: + - first_char +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/deprecation/manifest.yml b/filebeat/module/elasticsearch/deprecation/manifest.yml index a6332145175..8dfbaec866b 100644 --- a/filebeat/module/elasticsearch/deprecation/manifest.yml +++ b/filebeat/module/elasticsearch/deprecation/manifest.yml @@ -13,7 +13,7 @@ var: - c:/ProgramData/Elastic/Elasticsearch/logs/*_deprecation.json ingest_pipeline: - - ingest/pipeline.json - - ingest/pipeline-plaintext.json - - ingest/pipeline-json.json + - ingest/pipeline.yml + - ingest/pipeline-plaintext.yml + - ingest/pipeline-json.yml input: config/log.yml diff --git a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json index a1c8699c520..78e49516f93 100644 --- a/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json @@ -2,9 +2,12 @@ { "@timestamp": "2018-04-23T16:40:13.737-02:00", "elasticsearch.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -15,9 +18,12 @@ { "@timestamp": "2018-04-23T16:40:13.862-02:00", "elasticsearch.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -28,9 +34,12 @@ { "@timestamp": "2018-04-23T16:40:14.792-02:00", "elasticsearch.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -41,9 +50,12 @@ { "@timestamp": "2018-04-23T16:40:15.127-02:00", "elasticsearch.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", diff --git a/filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json index 79e81424205..bdb7e708946 100644 --- a/filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/other_elasticsearch_deprecation.log-expected.json @@ -2,9 +2,12 @@ { "@timestamp": "2017-11-30T13:38:16.911-02:00", "elasticsearch.component": "o.e.d.c.ParseField", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -15,9 +18,12 @@ { "@timestamp": "2017-11-30T13:38:16.941-02:00", "elasticsearch.component": "o.e.d.c.ParseField", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -28,9 +34,12 @@ { "@timestamp": "2017-11-30T13:39:28.986-02:00", "elasticsearch.component": "o.e.d.i.m.UidFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -41,9 +50,12 @@ { "@timestamp": "2017-11-30T13:39:36.339-02:00", "elasticsearch.component": "o.e.d.i.m.UidFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -54,9 +66,12 @@ { "@timestamp": "2017-11-30T13:40:49.540-02:00", "elasticsearch.component": "o.e.d.i.m.UidFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -67,9 +82,12 @@ { "@timestamp": "2017-11-30T14:08:37.413-02:00", "elasticsearch.component": "o.e.d.i.m.UidFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -80,9 +98,12 @@ { "@timestamp": "2017-11-30T14:08:37.413-02:00", "elasticsearch.component": "o.e.d.i.m.UidFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -93,9 +114,12 @@ { "@timestamp": "2017-11-30T14:08:46.006-02:00", "elasticsearch.component": "o.e.d.i.m.UidFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -106,9 +130,12 @@ { "@timestamp": "2017-11-30T14:08:46.006-02:00", "elasticsearch.component": "o.e.d.i.m.UidFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -119,9 +146,12 @@ { "@timestamp": "2017-12-01T14:05:54.017-02:00", "elasticsearch.component": "o.e.d.i.m.AllFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -132,9 +162,12 @@ { "@timestamp": "2017-12-01T14:05:54.019-02:00", "elasticsearch.component": "o.e.d.i.m.AllFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -145,9 +178,12 @@ { "@timestamp": "2017-12-01T14:06:52.059-02:00", "elasticsearch.component": "o.e.d.i.m.AllFieldMapper", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -158,9 +194,12 @@ { "@timestamp": "2017-12-01T14:46:10.428-02:00", "elasticsearch.component": "o.e.d.s.a.InternalOrder$Parser", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -171,9 +210,12 @@ { "@timestamp": "2017-12-04T16:17:18.271-02:00", "elasticsearch.component": "o.e.d.a.a.i.t.p.PutIndexTemplateRequest", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -184,9 +226,12 @@ { "@timestamp": "2017-12-04T16:17:18.282-02:00", "elasticsearch.component": "o.e.d.i.m.MapperService", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", @@ -197,9 +242,12 @@ { "@timestamp": "2017-12-04T16:20:43.248-02:00", "elasticsearch.component": "o.e.d.i.m.MapperService", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "deprecation", "input.type": "log", "log.level": "WARN", diff --git a/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json index 6f6b17316e4..446721df60c 100644 --- a/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json @@ -6,9 +6,13 @@ "elasticsearch.component": "o.e.d.r.a.d.RestGetAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 0, @@ -22,9 +26,13 @@ "elasticsearch.component": "o.e.d.a.b.BulkRequest", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 387, @@ -38,9 +46,13 @@ "elasticsearch.component": "o.e.d.r.a.d.RestUpdateAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 717, @@ -54,9 +66,13 @@ "elasticsearch.component": "o.e.d.r.a.s.RestSearchAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 1113, @@ -70,9 +86,13 @@ "elasticsearch.component": "o.e.d.x.s.r.a.u.RestChangePasswordAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 1452, @@ -86,9 +106,13 @@ "elasticsearch.component": "o.e.d.x.s.r.a.u.RestChangePasswordAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 1856, @@ -102,9 +126,13 @@ "elasticsearch.component": "o.e.d.x.s.r.a.u.RestChangePasswordAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 2260, @@ -118,9 +146,13 @@ "elasticsearch.component": "o.e.d.r.a.d.RestGetAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 2664, @@ -134,9 +166,13 @@ "elasticsearch.component": "o.e.d.a.b.BulkRequest", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 3051, @@ -150,9 +186,13 @@ "elasticsearch.component": "o.e.d.r.a.d.RestUpdateAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 3381, @@ -166,9 +206,13 @@ "elasticsearch.component": "o.e.d.r.a.s.RestSearchAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 3777, @@ -182,9 +226,13 @@ "elasticsearch.component": "o.e.d.x.w.a.i.IndexAction", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 4116, @@ -198,9 +246,13 @@ "elasticsearch.component": "o.e.d.i.q.QueryShardContext", "elasticsearch.node.id": "gCoNXf3qSQ6a190zBKr7Bw", "elasticsearch.node.name": "es1_1", + "event.category": "database", "event.dataset": "elasticsearch.deprecation", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "deprecation", + "host.id": "gCoNXf3qSQ6a190zBKr7Bw", "input.type": "log", "log.level": "WARN", "log.offset": 4459, diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.json b/filebeat/module/elasticsearch/gc/ingest/pipeline.json deleted file mode 100644 index 3dbc83bdfe2..00000000000 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "description": "Pipeline for parsing Elasticsearch JVM garbage collection logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "(?:%{JVM8HEADER}|%{JVM9HEADER}) Total time for which application threads were stopped: %{BASE10NUM:elasticsearch.gc.threads_total_stop_time_sec} seconds, Stopping threads took: %{BASE10NUM:elasticsearch.gc.stopping_threads_time_sec} seconds", - "(?:%{JVM8HEADER}) \\[GC \\(%{DATA:elasticsearch.gc.phase.name}\\) \\[YG occupancy: %{BASE10NUM:elasticsearch.gc.young_gen.used_kb} K \\(%{BASE10NUM:elasticsearch.gc.young_gen.size_kb} K\\)\\]%{BASE10NUM}: \\[Rescan \\(parallel\\) , %{BASE10NUM:elasticsearch.gc.phase.parallel_rescan_time_sec} secs\\]%{BASE10NUM}: \\[weak refs processing, %{BASE10NUM:elasticsearch.gc.phase.weak_refs_processing_time_sec} secs\\]%{BASE10NUM}: \\[class unloading, %{BASE10NUM:elasticsearch.gc.phase.class_unload_time_sec} secs\\]%{BASE10NUM}: \\[scrub symbol table, %{BASE10NUM:elasticsearch.gc.phase.scrub_symbol_table_time_sec} secs\\]%{BASE10NUM}: \\[scrub string table, %{BASE10NUM:elasticsearch.gc.phase.scrub_string_table_time_sec} secs\\]\\[1 CMS-remark: %{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\\)\\] %{BASE10NUM:elasticsearch.gc.heap.used_kb}K\\(%{BASE10NUM:elasticsearch.gc.heap.size_kb}K\\), %{BASE10NUM:elasticsearch.gc.phase.duration_sec} secs\\] %{PROCTIME}", - "(?:%{JVM8HEADER}) \\[GC \\(%{DATA:elasticsearch.gc.phase.name}\\) \\[%{BASE10NUM} CMS-initial-mark: %{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\\)\\] %{BASE10NUM:elasticsearch.gc.heap.used_kb}K\\(%{BASE10NUM:elasticsearch.gc.heap.size_kb}K\\), %{BASE10NUM:elasticsearch.gc.phase.duration_sec} secs\\] %{PROCTIME}", - "%{JVM9HEADER} GC\\(%{BASE10NUM}\\) ParNew: %{BASE10NUM}K-\\>%{BASE10NUM:elasticsearch.gc.young_gen.used_kb}K\\(%{BASE10NUM:elasticsearch.gc.young_gen.size_kb}K\\)", - "%{JVM9HEADER} GC\\(%{BASE10NUM}\\) Old: %{BASE10NUM}K-\\>%{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\\)", - "(?:%{JVM8HEADER}|%{JVM9HEADER}) %{GREEDYMULTILINE:message}" - ], - "pattern_definitions": { - "GREEDYMULTILINE": "(.|\n)*", - "JVM8HEADER": "%{TIMESTAMP_ISO8601:timestamp}: %{BASE10NUM:elasticsearch.gc.jvm_runtime_sec}:", - "JVM9HEADER": "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{POSINT:process.pid}\\]\\[%{DATA:elasticsearch.gc.tags}%{SPACE}\\]", - "PROCTIME": "\\[Times: user=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.user_sec} sys=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.sys_sec}, real=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.real_sec} secs\\]" - } - } - }, - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ] - } - }, - { - "remove": { - "field": "timestamp" - } - }, - { - "split": { - "field": "elasticsearch.gc.tags", - "separator": ",", - "ignore_missing": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml new file mode 100644 index 00000000000..fc8ec5c73e3 --- /dev/null +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml @@ -0,0 +1,62 @@ +description: Pipeline for parsing Elasticsearch JVM garbage collection logs +processors: +- grok: + field: message + patterns: + - '(?:%{JVM8HEADER}|%{JVM9HEADER}) Total time for which application threads were + stopped: %{BASE10NUM:elasticsearch.gc.threads_total_stop_time_sec} seconds, + Stopping threads took: %{BASE10NUM:elasticsearch.gc.stopping_threads_time_sec} + seconds' + - '(?:%{JVM8HEADER}) \[GC \(%{DATA:elasticsearch.gc.phase.name}\) \[YG occupancy: + %{BASE10NUM:elasticsearch.gc.young_gen.used_kb} K \(%{BASE10NUM:elasticsearch.gc.young_gen.size_kb} + K\)\]%{BASE10NUM}: \[Rescan \(parallel\) , %{BASE10NUM:elasticsearch.gc.phase.parallel_rescan_time_sec} + secs\]%{BASE10NUM}: \[weak refs processing, %{BASE10NUM:elasticsearch.gc.phase.weak_refs_processing_time_sec} + secs\]%{BASE10NUM}: \[class unloading, %{BASE10NUM:elasticsearch.gc.phase.class_unload_time_sec} + secs\]%{BASE10NUM}: \[scrub symbol table, %{BASE10NUM:elasticsearch.gc.phase.scrub_symbol_table_time_sec} + secs\]%{BASE10NUM}: \[scrub string table, %{BASE10NUM:elasticsearch.gc.phase.scrub_string_table_time_sec} + secs\]\[1 CMS-remark: %{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)\] + %{BASE10NUM:elasticsearch.gc.heap.used_kb}K\(%{BASE10NUM:elasticsearch.gc.heap.size_kb}K\), + %{BASE10NUM:elasticsearch.gc.phase.duration_sec} secs\] %{PROCTIME}' + - '(?:%{JVM8HEADER}) \[GC \(%{DATA:elasticsearch.gc.phase.name}\) \[%{BASE10NUM} + CMS-initial-mark: %{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)\] + %{BASE10NUM:elasticsearch.gc.heap.used_kb}K\(%{BASE10NUM:elasticsearch.gc.heap.size_kb}K\), + %{BASE10NUM:elasticsearch.gc.phase.duration_sec} secs\] %{PROCTIME}' + - '%{JVM9HEADER} GC\(%{BASE10NUM}\) ParNew: %{BASE10NUM}K-\>%{BASE10NUM:elasticsearch.gc.young_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.young_gen.size_kb}K\)' + - '%{JVM9HEADER} GC\(%{BASE10NUM}\) Old: %{BASE10NUM}K-\>%{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)' + - (?:%{JVM8HEADER}|%{JVM9HEADER}) %{GREEDYMULTILINE:message} + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + JVM8HEADER: '%{TIMESTAMP_ISO8601:timestamp}: %{BASE10NUM:elasticsearch.gc.jvm_runtime_sec}:' + JVM9HEADER: \[%{TIMESTAMP_ISO8601:timestamp}\]\[%{POSINT:process.pid}\]\[%{DATA:elasticsearch.gc.tags}%{SPACE}\] + PROCTIME: '\[Times: user=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.user_sec} + sys=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.sys_sec}, real=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.real_sec} + secs\]' +- rename: + field: '@timestamp' + target_field: event.created +- date: + field: timestamp + target_field: '@timestamp' + formats: + - ISO8601 +- remove: + field: timestamp +- set: + field: event.kind + value: metric +- set: + field: event.category + value: database +- set: + field: event.type + value: info +- split: + field: elasticsearch.gc.tags + separator: ',' + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/gc/manifest.yml b/filebeat/module/elasticsearch/gc/manifest.yml index 64d84c3c82a..3add81562cc 100644 --- a/filebeat/module/elasticsearch/gc/manifest.yml +++ b/filebeat/module/elasticsearch/gc/manifest.yml @@ -12,5 +12,5 @@ var: - c:/ProgramData/Elastic/Elasticsearch/logs/gc.log.* - c:/ProgramData/Elastic/Elasticsearch/logs/gc.log -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/gc.yml diff --git a/filebeat/module/elasticsearch/gc/test/gc.log-expected.json b/filebeat/module/elasticsearch/gc/test/gc.log-expected.json index 8eacfe198b4..41943abdf00 100644 --- a/filebeat/module/elasticsearch/gc/test/gc.log-expected.json +++ b/filebeat/module/elasticsearch/gc/test/gc.log-expected.json @@ -4,8 +4,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 0, @@ -20,8 +23,11 @@ "heap", "coops" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 70, @@ -34,8 +40,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 201, @@ -48,8 +57,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 290, @@ -62,8 +74,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 458, @@ -76,8 +91,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 547, @@ -90,8 +108,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 715, @@ -104,8 +125,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 804, @@ -118,8 +142,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 972, @@ -132,8 +159,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 1061, @@ -146,8 +176,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 1229, @@ -160,8 +193,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 1318, @@ -174,8 +210,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 1486, @@ -188,8 +227,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 1575, @@ -202,8 +244,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 1743, @@ -216,8 +261,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 1832, @@ -230,8 +278,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 2000, @@ -244,8 +295,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 2089, @@ -258,8 +312,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 2257, @@ -272,8 +329,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 2346, @@ -286,8 +346,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 2514, @@ -300,8 +363,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 2603, @@ -314,8 +380,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 2771, @@ -328,8 +397,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 2860, @@ -342,8 +414,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 3028, @@ -356,8 +431,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 3117, @@ -370,8 +448,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 3285, @@ -384,8 +465,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 3374, @@ -398,8 +482,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 3542, @@ -412,8 +499,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 3631, @@ -426,8 +516,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 3799, @@ -440,8 +533,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 3888, @@ -454,8 +550,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 4056, @@ -468,8 +567,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 4145, @@ -482,8 +584,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 4313, @@ -496,8 +601,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 4402, @@ -510,8 +618,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 4570, @@ -524,8 +635,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 4659, @@ -538,8 +652,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 4827, @@ -552,8 +669,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 4916, @@ -566,8 +686,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5084, @@ -581,8 +704,11 @@ "gc", "start" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5173, @@ -596,8 +722,11 @@ "gc", "task" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5265, @@ -611,8 +740,11 @@ "gc", "age" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5360, @@ -626,8 +758,11 @@ "gc", "age" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5491, @@ -641,8 +776,11 @@ "gc", "age" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5595, @@ -658,8 +796,11 @@ ], "elasticsearch.gc.young_gen.size_kb": "314560", "elasticsearch.gc.young_gen.used_kb": "17562", + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5700, @@ -673,8 +814,11 @@ "gc", "heap" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5792, @@ -688,8 +832,11 @@ "gc", "metaspace" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5872, @@ -702,8 +849,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 5967, @@ -717,8 +867,11 @@ "gc", "cpu" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6085, @@ -731,8 +884,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6176, @@ -745,8 +901,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6344, @@ -760,8 +919,11 @@ "gc", "start" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6433, @@ -774,8 +936,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6511, @@ -789,8 +954,11 @@ "gc", "cpu" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6612, @@ -803,8 +971,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6703, @@ -817,8 +988,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6871, @@ -832,8 +1006,11 @@ "gc", "task" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 6946, @@ -846,8 +1023,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7038, @@ -861,8 +1041,11 @@ "gc", "cpu" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7121, @@ -875,8 +1058,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7212, @@ -889,8 +1075,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7291, @@ -904,8 +1093,11 @@ "gc", "cpu" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7378, @@ -918,8 +1110,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7469, @@ -932,8 +1127,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7558, @@ -946,8 +1144,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7647, @@ -960,8 +1161,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7815, @@ -975,8 +1179,11 @@ "gc", "cpu" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 7914, @@ -989,8 +1196,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8005, @@ -1004,8 +1214,11 @@ "gc", "start" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8094, @@ -1018,8 +1231,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8166, @@ -1033,8 +1249,11 @@ "gc", "cpu" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8264, @@ -1047,8 +1266,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8355, @@ -1061,8 +1283,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8523, @@ -1075,8 +1300,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8599, @@ -1090,8 +1318,11 @@ "gc", "cpu" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8683, @@ -1104,8 +1335,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8774, @@ -1118,8 +1352,11 @@ "elasticsearch.gc.tags": [ "gc" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8850, @@ -1133,8 +1370,11 @@ "gc", "cpu" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 8934, @@ -1150,8 +1390,11 @@ "gc", "heap" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9025, @@ -1164,8 +1407,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9105, @@ -1178,8 +1424,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9194, @@ -1192,8 +1441,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9362, @@ -1206,8 +1458,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9451, @@ -1220,8 +1475,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9619, @@ -1234,8 +1492,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9708, @@ -1248,8 +1509,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9876, @@ -1262,8 +1526,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 9965, @@ -1276,8 +1543,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 10133, @@ -1290,8 +1560,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 10222, @@ -1304,8 +1577,11 @@ "elasticsearch.gc.tags": [ "safepoint" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 10390, @@ -1319,8 +1595,11 @@ "gc", "start" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 10479, @@ -1334,8 +1613,11 @@ "gc", "task" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 10571, @@ -1349,8 +1631,11 @@ "gc", "age" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 10666, @@ -1364,8 +1649,11 @@ "gc", "age" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 10797, @@ -1379,8 +1667,11 @@ "gc", "age" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 10901, @@ -1394,8 +1685,11 @@ "gc", "age" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 11006, @@ -1411,8 +1705,11 @@ ], "elasticsearch.gc.young_gen.size_kb": "314560", "elasticsearch.gc.young_gen.used_kb": "25722", + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 11111, @@ -1426,8 +1723,11 @@ "gc", "heap" ], + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 11203, diff --git a/filebeat/module/elasticsearch/gc/test/test.log-expected.json b/filebeat/module/elasticsearch/gc/test/test.log-expected.json index 523dc237581..b0bcf5ecd0b 100644 --- a/filebeat/module/elasticsearch/gc/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/gc/test/test.log-expected.json @@ -11,8 +11,11 @@ "elasticsearch.gc.phase.cpu_time.user_sec": "0.01", "elasticsearch.gc.phase.duration_sec": "0.0021716", "elasticsearch.gc.phase.name": "CMS Initial Mark", + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 0, @@ -24,8 +27,11 @@ "elasticsearch.gc.jvm_runtime_sec": "1396138.752", "elasticsearch.gc.stopping_threads_time_sec": "0.0000702", "elasticsearch.gc.threads_total_stop_time_sec": "0.0083760", + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 181, @@ -51,8 +57,11 @@ "elasticsearch.gc.phase.weak_refs_processing_time_sec": "0.0003647", "elasticsearch.gc.young_gen.size_kb": "157248", "elasticsearch.gc.young_gen.used_kb": "113198", + "event.category": "database", "event.dataset": "elasticsearch.gc", + "event.kind": "metric", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "gc", "input.type": "log", "log.offset": 339, diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.json b/filebeat/module/elasticsearch/server/ingest/pipeline-json.json deleted file mode 100644 index ada7c5063f4..00000000000 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-json.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "description": "Pipeline for parsing the Elasticsearch server log file in JSON format.", - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ], - "processors": [ - { - "json": { - "field": "message", - "target_field": "elasticsearch.server" - } - }, - { - "drop": { - "if": "ctx.elasticsearch.server.type != 'server'" - } - }, - { - "remove": { - "field": "elasticsearch.server.type" - } - }, - { - "rename": { - "field": "elasticsearch.server.level", - "target_field": "log.level" - } - }, - { - "rename": { - "field": "elasticsearch.server.component", - "target_field": "elasticsearch.component" - } - }, - { - "dot_expander": { - "field": "cluster.name", - "path": "elasticsearch.server" - } - }, - { - "rename": { - "field": "elasticsearch.server.cluster.name", - "target_field": "elasticsearch.cluster.name" - } - }, - { - "dot_expander": { - "field": "node.name", - "path": "elasticsearch.server" - } - }, - { - "rename": { - "field": "elasticsearch.server.node.name", - "target_field": "elasticsearch.node.name" - } - }, - { - "dot_expander": { - "field": "cluster.uuid", - "path": "elasticsearch.server" - } - }, - { - "rename": { - "field": "elasticsearch.server.cluster.uuid", - "target_field": "elasticsearch.cluster.uuid", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "node.id", - "path": "elasticsearch.server" - } - }, - { - "rename": { - "field": "elasticsearch.server.node.id", - "target_field": "elasticsearch.node.id", - "ignore_missing": true - } - }, - { - "grok": { - "field": "elasticsearch.server.message", - "pattern_definitions": { - "GREEDYMULTILINE": "(.|\n)*", - "INDEXNAME": "[a-zA-Z0-9_.-]*", - "GC_ALL": "\\[gc\\]\\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\\] overhead, spent \\[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\\] collecting in the last \\[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\\]", - "GC_YOUNG": "\\[gc\\]\\[young\\]\\[%{NUMBER:elasticsearch.server.gc.young.one}\\]\\[%{NUMBER:elasticsearch.server.gc.young.two}\\]%{SPACE}%{GREEDYMULTILINE:message}" - }, - "patterns": [ - "%{GC_ALL}", - "%{GC_YOUNG}", - "((\\[%{INDEXNAME:elasticsearch.index.name}\\]|\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\]))?%{SPACE}%{GREEDYMULTILINE:message}" - ] - } - }, - { - "remove": { - "field": "elasticsearch.server.message" - } - }, - { - "date": { - "field": "elasticsearch.server.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - "ignore_failure": true - } - } - ] -} diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml new file mode 100644 index 00000000000..1f2022f0b65 --- /dev/null +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml @@ -0,0 +1,68 @@ +description: Pipeline for parsing the Elasticsearch server log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + target_field: elasticsearch.server +- drop: + if: ctx.elasticsearch.server.type != 'server' +- remove: + field: elasticsearch.server.type +- rename: + field: elasticsearch.server.level + target_field: log.level +- rename: + field: elasticsearch.server.component + target_field: elasticsearch.component +- dot_expander: + field: cluster.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.server +- rename: + field: elasticsearch.server.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.server +- rename: + field: elasticsearch.server.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- grok: + field: elasticsearch.server.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + GC_ALL: \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent + \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] + collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] + GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} + patterns: + - '%{GC_ALL}' + - '%{GC_YOUNG}' + - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} +- remove: + field: elasticsearch.server.message +- date: + field: elasticsearch.server.timestamp + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json deleted file mode 100755 index 0da534584b3..00000000000 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "description": "Pipeline for parsing the Elasticsearch server log file in plaintext format.", - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ], - "processors": [ - { - "grok": { - "field": "message", - "pattern_definitions": { - "GREEDYMULTILINE": "(.|\n)*", - "INDEXNAME": "[a-zA-Z0-9_.-]*", - "GC_ALL": "\\[gc\\]\\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\\] overhead, spent \\[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\\] collecting in the last \\[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\\]", - "GC_YOUNG": "\\[gc\\]\\[young\\]\\[%{NUMBER:elasticsearch.server.gc.young.one}\\]\\[%{NUMBER:elasticsearch.server.gc.young.two}\\]%{SPACE}%{GREEDYMULTILINE:message}", - "LOG_HEADER": "\\[%{TIMESTAMP_ISO8601:elasticsearch.server.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}\\]\\[%{DATA:elasticsearch.component}%{SPACE}\\](%{SPACE})?(\\[%{DATA:elasticsearch.node.name}\\])?(%{SPACE})?" - }, - "patterns": [ - "%{LOG_HEADER}%{GC_ALL}", - "%{LOG_HEADER}%{GC_YOUNG}", - "%{LOG_HEADER}%{SPACE}((\\[%{INDEXNAME:elasticsearch.index.name}\\]|\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\]))?%{SPACE}%{GREEDYMULTILINE:message}" - ] - } - }, - { - "date": { - "if": "ctx.event.timezone == null", - "field": "elasticsearch.server.timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "elasticsearch.server.timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - } - ] -} diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.yml new file mode 100644 index 00000000000..91570b652d3 --- /dev/null +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.yml @@ -0,0 +1,43 @@ +description: Pipeline for parsing the Elasticsearch server log file in plaintext format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- grok: + field: message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + GC_ALL: \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent + \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] + collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] + GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} + LOG_HEADER: \[%{TIMESTAMP_ISO8601:elasticsearch.server.timestamp}\]\[%{LOGLEVEL:log.level}%{SPACE}\]\[%{DATA:elasticsearch.component}%{SPACE}\](%{SPACE})?(\[%{DATA:elasticsearch.node.name}\])?(%{SPACE})? + patterns: + - '%{LOG_HEADER}%{GC_ALL}' + - '%{LOG_HEADER}%{GC_YOUNG}' + - '%{LOG_HEADER}%{SPACE}((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message}' +- date: + if: ctx.event.timezone == null + field: elasticsearch.server.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx.event.timezone != null + field: elasticsearch.server.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.json b/filebeat/module/elasticsearch/server/ingest/pipeline.json deleted file mode 100644 index de23917aa96..00000000000 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "description": "Pipeline for parsing elasticsearch server logs", - "processors": [ - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "grok": { - "field": "message", - "patterns": [ - "^%{CHAR:first_char}" - ], - "pattern_definitions": { - "CHAR": "." - } - } - }, - { - "pipeline": { - "if": "ctx.first_char != '{'", - "name": "{< IngestPipeline "pipeline-plaintext" >}" - } - }, - { - "pipeline": { - "if": "ctx.first_char == '{'", - "name": "{< IngestPipeline "pipeline-json" >}" - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.observation_duration != null) { if (ctx.elasticsearch.server.gc.observation_duration.unit == params.seconds_unit) { ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_s;}if (ctx.elasticsearch.server.gc.observation_duration.unit == params.milliseconds_unit) { ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time; } if (ctx.elasticsearch.server.gc.observation_duration.unit == params.minutes_unit) { ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_m; }} if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.collection_duration != null) { if (ctx.elasticsearch.server.gc.collection_duration.unit == params.seconds_unit) { ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_s;} if (ctx.elasticsearch.server.gc.collection_duration.unit == params.milliseconds_unit) {ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time; } if (ctx.elasticsearch.server.gc.collection_duration.unit == params.minutes_unit) { ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_m; }}", - "params": { - "minutes_unit": "m", - "seconds_unit": "s", - "milliseconds_unit": "ms", - "ms_in_one_s": 1000, - "ms_in_one_m": 60000 - } - } - }, - { - "remove": { - "field": [ - "elasticsearch.server.gc.collection_duration.time", - "elasticsearch.server.gc.collection_duration.unit", - "elasticsearch.server.gc.observation_duration.time", - "elasticsearch.server.gc.observation_duration.unit" - ], - "ignore_missing": true - } - }, - { - "remove": { - "field": "elasticsearch.server.timestamp" - } - }, - { - "remove": { - "field": [ - "first_char" - ] - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml new file mode 100644 index 00000000000..032e3581d0b --- /dev/null +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -0,0 +1,90 @@ +description: Pipeline for parsing elasticsearch server logs +processors: +- rename: + field: '@timestamp' + target_field: event.created +- grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . +- pipeline: + if: ctx.first_char != '{' + name: '{< IngestPipeline "pipeline-plaintext" >}' +- pipeline: + if: ctx.first_char == '{' + name: '{< IngestPipeline "pipeline-json" >}' +- script: + lang: painless + source: >- + if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.observation_duration != null) { + if (ctx.elasticsearch.server.gc.observation_duration.unit == params.seconds_unit) { + ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_s; + } + if (ctx.elasticsearch.server.gc.observation_duration.unit == params.milliseconds_unit) { + ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time; + } + if (ctx.elasticsearch.server.gc.observation_duration.unit == params.minutes_unit) { + ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_m; + } + } + if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.collection_duration != null) { + if (ctx.elasticsearch.server.gc.collection_duration.unit == params.seconds_unit) { + ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_s; + } + if (ctx.elasticsearch.server.gc.collection_duration.unit == params.milliseconds_unit) { + ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time; + } + if (ctx.elasticsearch.server.gc.collection_duration.unit == params.minutes_unit) { + ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_m; + } + } + params: + minutes_unit: m + seconds_unit: s + milliseconds_unit: ms + ms_in_one_s: 1000 + ms_in_one_m: 60000 + +- set: + field: event.kind + value: event +- set: + field: event.category + value: database +- script: + lang: painless + source: >- + def errorLevels = ['FATAL', 'ERROR']; + if (ctx?.log?.level != null) { + if (errorLevels.contains(ctx.log.level)) { + ctx.event.type = 'error'; + } else { + ctx.event.type = 'info'; + } + } +- set: + field: host.name + value: "{{elasticsearch.node.name}}" + if: "ctx?.elasticsearch?.node?.name != null" +- set: + field: host.id + value: "{{elasticsearch.node.id}}" + if: "ctx?.elasticsearch?.node?.id != null" +- remove: + field: + - elasticsearch.server.gc.collection_duration.time + - elasticsearch.server.gc.collection_duration.unit + - elasticsearch.server.gc.observation_duration.time + - elasticsearch.server.gc.observation_duration.unit + ignore_missing: true +- remove: + field: elasticsearch.server.timestamp +- remove: + field: + - first_char +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/server/manifest.yml b/filebeat/module/elasticsearch/server/manifest.yml index ae0399a8690..406972cba56 100644 --- a/filebeat/module/elasticsearch/server/manifest.yml +++ b/filebeat/module/elasticsearch/server/manifest.yml @@ -13,7 +13,7 @@ var: - c:/ProgramData/Elastic/Elasticsearch/logs/*_server.json ingest_pipeline: - - ingest/pipeline.json - - ingest/pipeline-plaintext.json - - ingest/pipeline-json.json + - ingest/pipeline.yml + - ingest/pipeline-plaintext.yml + - ingest/pipeline-json.yml input: config/log.yml diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch.624.log-expected.json b/filebeat/module/elasticsearch/server/test/elasticsearch.624.log-expected.json index 436b0229046..11e77e6a3c9 100644 --- a/filebeat/module/elasticsearch/server/test/elasticsearch.624.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/elasticsearch.624.log-expected.json @@ -3,9 +3,12 @@ "@timestamp": "2018-05-17T08:19:35.939-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -17,9 +20,12 @@ "@timestamp": "2018-05-17T08:19:36.089-02:00", "elasticsearch.component": "o.e.e.NodeEnvironment", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -31,9 +37,12 @@ "@timestamp": "2018-05-17T08:19:36.090-02:00", "elasticsearch.component": "o.e.e.NodeEnvironment", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -44,9 +53,12 @@ { "@timestamp": "2018-05-17T08:19:36.116-02:00", "elasticsearch.component": "o.e.n.Node", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -57,9 +69,12 @@ { "@timestamp": "2018-05-17T08:19:36.117-02:00", "elasticsearch.component": "o.e.n.Node", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -70,9 +85,12 @@ { "@timestamp": "2018-05-17T08:19:36.117-02:00", "elasticsearch.component": "o.e.n.Node", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -84,9 +102,12 @@ "@timestamp": "2018-05-17T08:19:37.563-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -98,9 +119,12 @@ "@timestamp": "2018-05-17T08:19:37.564-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -112,9 +136,12 @@ "@timestamp": "2018-05-17T08:19:37.564-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -126,9 +153,12 @@ "@timestamp": "2018-05-17T08:19:37.564-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -140,9 +170,12 @@ "@timestamp": "2018-05-17T08:19:37.564-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -154,9 +187,12 @@ "@timestamp": "2018-05-17T08:19:37.564-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -168,9 +204,12 @@ "@timestamp": "2018-05-17T08:19:37.565-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -182,9 +221,12 @@ "@timestamp": "2018-05-17T08:19:37.565-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -196,9 +238,12 @@ "@timestamp": "2018-05-17T08:19:37.565-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -210,9 +255,12 @@ "@timestamp": "2018-05-17T08:19:37.565-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -224,9 +272,12 @@ "@timestamp": "2018-05-17T08:19:37.566-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -238,9 +289,12 @@ "@timestamp": "2018-05-17T08:19:37.566-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -252,9 +306,12 @@ "@timestamp": "2018-05-17T08:19:37.566-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -266,9 +323,12 @@ "@timestamp": "2018-05-17T08:19:37.566-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -280,9 +340,12 @@ "@timestamp": "2018-05-17T08:19:37.567-02:00", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -294,9 +357,12 @@ "@timestamp": "2018-05-17T08:19:43.741-02:00", "elasticsearch.component": "o.e.d.DiscoveryModule", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -307,9 +373,12 @@ { "@timestamp": "2018-05-17T08:19:45.090-02:00", "elasticsearch.component": "o.e.n.Node", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -321,9 +390,12 @@ "@timestamp": "2018-05-17T08:19:45.090-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -335,9 +407,12 @@ "@timestamp": "2018-05-17T08:19:45.482-02:00", "elasticsearch.component": "o.e.t.TransportService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -349,9 +424,12 @@ "@timestamp": "2018-05-17T08:19:48.816-02:00", "elasticsearch.component": "o.e.c.s.MasterService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -363,9 +441,12 @@ "@timestamp": "2018-05-17T08:19:48.826-02:00", "elasticsearch.component": "o.e.c.s.ClusterApplierService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -377,9 +458,12 @@ "@timestamp": "2018-05-17T08:19:48.895-02:00", "elasticsearch.component": "o.e.h.n.Netty4HttpServerTransport", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -391,9 +475,12 @@ "@timestamp": "2018-05-17T08:19:48.895-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -405,9 +492,12 @@ "@timestamp": "2018-05-17T08:19:49.354-02:00", "elasticsearch.component": "o.e.g.GatewayService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -419,9 +509,12 @@ "@timestamp": "2018-05-17T08:19:50.077-02:00", "elasticsearch.component": "o.e.c.r.a.AllocationService", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -433,9 +526,12 @@ "@timestamp": "2018-05-17T08:20:18.871-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -448,9 +544,12 @@ "elasticsearch.component": "o.e.c.m.MetaDataCreateIndexService", "elasticsearch.index.name": "metricbeat-7.0.0-alpha1-2018.05.17", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -462,9 +561,12 @@ "@timestamp": "2018-05-17T08:20:48.886-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -476,9 +578,12 @@ "@timestamp": "2018-05-17T08:21:18.895-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -490,9 +595,12 @@ "@timestamp": "2018-05-17T08:21:48.904-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -504,9 +612,12 @@ "@timestamp": "2018-05-17T08:22:18.911-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -518,9 +629,12 @@ "@timestamp": "2018-05-17T08:22:48.920-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -532,9 +646,12 @@ "@timestamp": "2018-05-17T08:23:18.932-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -546,9 +663,12 @@ "@timestamp": "2018-05-17T08:23:48.941-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -560,9 +680,12 @@ "@timestamp": "2018-05-17T08:24:18.956-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -574,9 +697,12 @@ "@timestamp": "2018-05-17T08:24:48.963-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -588,9 +714,12 @@ "@timestamp": "2018-05-17T08:25:18.976-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -602,9 +731,12 @@ "@timestamp": "2018-05-17T08:25:48.988-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -616,9 +748,12 @@ "@timestamp": "2018-05-17T08:26:18.997-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -630,9 +765,12 @@ "@timestamp": "2018-05-17T08:26:49.009-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -644,9 +782,12 @@ "@timestamp": "2018-05-17T08:27:19.024-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -658,9 +799,12 @@ "@timestamp": "2018-05-17T08:27:49.035-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -672,9 +816,12 @@ "@timestamp": "2018-05-17T08:28:19.048-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -686,9 +833,12 @@ "@timestamp": "2018-05-17T08:28:49.060-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -701,9 +851,12 @@ "elasticsearch.component": "o.e.c.m.MetaDataCreateIndexService", "elasticsearch.index.name": "filebeat-test-input", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -717,9 +870,12 @@ "elasticsearch.index.id": "aOGgDwbURfCV57AScqbCgw", "elasticsearch.index.name": "filebeat-test-input", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -732,9 +888,12 @@ "elasticsearch.component": "o.e.c.m.MetaDataCreateIndexService", "elasticsearch.index.name": "test-filebeat-modules", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -748,9 +907,12 @@ "elasticsearch.index.id": "npNY8YrBQtC7JpFOh1sB0w", "elasticsearch.index.name": "test-filebeat-modules", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -762,9 +924,12 @@ "@timestamp": "2018-05-17T08:29:19.114-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -776,9 +941,12 @@ "@timestamp": "2018-05-17T08:29:25.418-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -790,9 +958,12 @@ "@timestamp": "2018-05-17T08:29:25.598-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -804,9 +975,12 @@ "@timestamp": "2018-05-17T08:29:25.598-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -818,9 +992,12 @@ "@timestamp": "2018-05-17T08:29:25.612-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", diff --git a/filebeat/module/elasticsearch/server/test/test-json.log-expected.json b/filebeat/module/elasticsearch/server/test/test-json.log-expected.json index 10949d3b4ba..e0b676a3df3 100644 --- a/filebeat/module/elasticsearch/server/test/test-json.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/test-json.log-expected.json @@ -4,8 +4,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.e.NodeEnvironment", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -18,8 +21,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.e.NodeEnvironment", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -32,8 +38,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -46,8 +55,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -60,8 +72,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -74,8 +89,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "WARN", @@ -88,8 +106,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -102,8 +123,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -116,8 +140,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -130,8 +157,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -144,8 +174,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -158,8 +191,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -172,8 +208,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -186,8 +225,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -200,8 +242,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -214,8 +259,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -228,8 +276,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -242,8 +293,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -256,8 +310,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -270,8 +327,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -284,8 +344,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -298,8 +361,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -312,8 +378,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -326,8 +395,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -340,8 +412,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -354,8 +429,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -368,8 +446,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -382,8 +463,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -396,8 +480,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -410,8 +497,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -424,8 +514,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -438,8 +531,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -452,8 +548,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -466,8 +565,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -480,8 +582,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.p.PluginsService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -494,8 +599,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.x.s.a.s.FileRolesStore", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -510,8 +618,11 @@ "elasticsearch.index.id": "25947", "elasticsearch.index.name": "controller", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -524,8 +635,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.a.ActionModule", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "DEBUG", @@ -538,8 +652,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.d.DiscoveryModule", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -552,8 +669,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -566,8 +686,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -580,8 +703,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.t.TransportService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -594,8 +720,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.d.z.FileBasedUnicastHostsProvider", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "WARN", @@ -608,8 +737,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.c.s.MasterService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -622,8 +754,11 @@ "elasticsearch.cluster.name": "distribution_run", "elasticsearch.component": "o.e.c.s.ClusterApplierService", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -638,9 +773,13 @@ "elasticsearch.component": "o.e.h.AbstractHttpServerTransport", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 12847, @@ -654,9 +793,13 @@ "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 13214, @@ -670,9 +813,13 @@ "elasticsearch.component": "o.e.l.LicenseService", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 13485, @@ -686,9 +833,13 @@ "elasticsearch.component": "o.e.g.GatewayService", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 13826, @@ -702,9 +853,13 @@ "elasticsearch.component": "o.e.x.m.p.NativeController", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 14140, @@ -718,9 +873,13 @@ "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 14498, @@ -734,9 +893,13 @@ "elasticsearch.component": "o.e.x.w.WatcherService", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 14774, @@ -750,9 +913,13 @@ "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 15101, @@ -766,9 +933,13 @@ "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 15372, @@ -782,9 +953,13 @@ "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.id": "WFhM7OPMScyc-25-pLM3cw", "elasticsearch.node.name": "node-0", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", + "host.id": "WFhM7OPMScyc-25-pLM3cw", "input.type": "log", "log.level": "INFO", "log.offset": 15647, @@ -845,8 +1020,11 @@ "at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368) [randomizedtesting-runner-2.7.1.jar:?]", "at java.lang.Thread.run(Thread.java:834) [?:?]" ], + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "error", "fileset.name": "server", "input.type": "log", "log.flags": [ @@ -862,8 +1040,11 @@ "elasticsearch.cluster.name": "elasticsearch", "elasticsearch.component": "o.e.c.l.JsonLoggerTests", "elasticsearch.node.name": "sample-name", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -918,8 +1099,11 @@ "Caused by: java.lang.RuntimeException: cause message", "... 37 more" ], + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "error", "fileset.name": "server", "input.type": "log", "log.flags": [ diff --git a/filebeat/module/elasticsearch/server/test/test.log-expected.json b/filebeat/module/elasticsearch/server/test/test.log-expected.json index 548642d15c3..ae7cf73df57 100644 --- a/filebeat/module/elasticsearch/server/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/test.log-expected.json @@ -4,9 +4,12 @@ "elasticsearch.component": "o.e.c.m.MetaDataCreateIndexService", "elasticsearch.index.name": "test-filebeat-modules", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -18,9 +21,12 @@ "@timestamp": "2018-05-17T08:19:35.939-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -32,9 +38,12 @@ "@timestamp": "2018-05-17T08:19:36.089-02:00", "elasticsearch.component": "o.e.e.NodeEnvironment", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -46,9 +55,12 @@ "@timestamp": "2018-05-17T08:19:36.090-02:00", "elasticsearch.component": "o.e.e.NodeEnvironment", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -59,9 +71,12 @@ { "@timestamp": "2018-05-17T08:19:36.116-02:00", "elasticsearch.component": "o.e.n.Node", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -73,9 +88,12 @@ "@timestamp": "2018-05-17T08:23:48.941-02:00", "elasticsearch.component": "o.e.c.r.a.DiskThresholdMonitor", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -88,9 +106,12 @@ "elasticsearch.component": "o.e.c.m.MetaDataCreateIndexService", "elasticsearch.index.name": "filebeat-test-input", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -104,9 +125,12 @@ "elasticsearch.index.id": "aOGgDwbURfCV57AScqbCgw", "elasticsearch.index.name": "filebeat-test-input", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -120,9 +144,12 @@ "elasticsearch.index.id": "3tWftqb4RLKdyCAga9syGA", "elasticsearch.index.name": ".kibana", "elasticsearch.node.name": "QGY1F5P", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -134,9 +161,12 @@ "@timestamp": "2018-05-17T08:29:25.598-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -148,9 +178,12 @@ "@timestamp": "2018-05-17T08:29:25.612-02:00", "elasticsearch.component": "o.e.n.Node", "elasticsearch.node.name": "vWNJsZ3", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -162,9 +195,12 @@ "@timestamp": "2018-07-03T11:45:48.548-02:00", "elasticsearch.component": "o.e.d.z.ZenDiscovery", "elasticsearch.node.name": "srvmulpvlsk252_md", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "INFO", @@ -176,9 +212,12 @@ "@timestamp": "2018-07-03T11:45:48.548-02:00", "elasticsearch.component": "o.e.d.z.ZenDiscovery", "elasticsearch.node.name": "srvmulpvlsk252_md", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.flags": [ @@ -192,9 +231,12 @@ { "@timestamp": "2018-07-03T11:45:52.666-02:00", "elasticsearch.component": "r.suppressed", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.flags": [ @@ -208,9 +250,12 @@ { "@timestamp": "2018-07-03T11:48:02.552-02:00", "elasticsearch.component": "r.suppressed", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.flags": [ @@ -227,9 +272,12 @@ "elasticsearch.node.name": "srvmulpvlsk252_md", "elasticsearch.server.gc.young.one": "3449979", "elasticsearch.server.gc.young.two": "986594", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.flags": [ @@ -247,9 +295,12 @@ "elasticsearch.server.gc.collection_duration.ms": 1600.0, "elasticsearch.server.gc.observation_duration.ms": 1800.0, "elasticsearch.server.gc.overhead_seq": "3449992", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "WARN", @@ -261,9 +312,12 @@ "@timestamp": "2018-07-03T11:48:02.541-02:00", "elasticsearch.component": "o.e.a.b.TransportShardBulkAction", "elasticsearch.node.name": "srvmulpvlsk252_md", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.level": "WARN", @@ -275,9 +329,12 @@ "@timestamp": "2018-07-03T20:10:07.376-02:00", "elasticsearch.component": "o.e.x.m.MonitoringService", "elasticsearch.node.name": "srvmulpvlsk252_md", + "event.category": "database", "event.dataset": "elasticsearch.server", + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "server", "input.type": "log", "log.flags": [ diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json deleted file mode 100644 index d37d170cf43..00000000000 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "description": "Pipeline for parsing the Elasticsearch slow logs in JSON format.", - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ], - "processors": [ - { - "json": { - "field": "message", - "target_field": "elasticsearch.slowlog" - } - }, - { - "drop": { - "if": "ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type != 'index_search_slowlog'" - } - }, - { - "remove": { - "field": "elasticsearch.slowlog.type" - } - }, - { - "rename": { - "field": "elasticsearch.slowlog.level", - "target_field": "log.level" - } - }, - { - "rename": { - "field": "elasticsearch.slowlog.component", - "target_field": "elasticsearch.component" - } - }, - { - "dot_expander": { - "field": "cluster.name", - "path": "elasticsearch.slowlog" - } - }, - { - "rename": { - "field": "elasticsearch.slowlog.cluster.name", - "target_field": "elasticsearch.cluster.name" - } - }, - { - "dot_expander": { - "field": "node.name", - "path": "elasticsearch.slowlog" - } - }, - { - "rename": { - "field": "elasticsearch.slowlog.node.name", - "target_field": "elasticsearch.node.name" - } - }, - { - "dot_expander": { - "field": "cluster.uuid", - "path": "elasticsearch.slowlog" - } - }, - { - "rename": { - "field": "elasticsearch.slowlog.cluster.uuid", - "target_field": "elasticsearch.cluster.uuid", - "ignore_missing": true - } - }, - { - "dot_expander": { - "field": "node.id", - "path": "elasticsearch.slowlog" - } - }, - { - "rename": { - "field": "elasticsearch.slowlog.node.id", - "target_field": "elasticsearch.node.id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "elasticsearch.slowlog.doc_type", - "target_field": "elasticsearch.slowlog.types", - "ignore_missing": true - } - }, - { - "convert": { - "field": "elasticsearch.slowlog.took_millis", - "type": "float", - "ignore_missing": true - } - }, - { - "rename": { - "field": "elasticsearch.slowlog.took_millis", - "target_field": "elasticsearch.slowlog.duration", - "ignore_missing": true - } - }, - { - "grok": { - "field": "elasticsearch.slowlog.message", - "pattern_definitions" : { - "GREEDYMULTILINE" : "(.|\n)*", - "INDEXNAME": "[a-zA-Z0-9_.-]*" - }, - "patterns": [ - "(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?", - "\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\]" - ] - } - }, - { - "remove": { - "field": "elasticsearch.slowlog.message" - } - }, - { - "date": { - "field": "elasticsearch.slowlog.timestamp", - "target_field": "@timestamp", - "formats": [ - "ISO8601" - ], - "ignore_failure": true - } - } - ] -} diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml new file mode 100644 index 00000000000..4c9b2c266d3 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -0,0 +1,76 @@ +description: Pipeline for parsing the Elasticsearch slow logs in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + target_field: elasticsearch.slowlog +- drop: + if: ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type + != 'index_search_slowlog' +- remove: + field: elasticsearch.slowlog.type +- rename: + field: elasticsearch.slowlog.level + target_field: log.level +- rename: + field: elasticsearch.slowlog.component + target_field: elasticsearch.component +- dot_expander: + field: cluster.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- rename: + field: elasticsearch.slowlog.doc_type + target_field: elasticsearch.slowlog.types + ignore_missing: true +- convert: + field: elasticsearch.slowlog.took_millis + type: float + ignore_missing: true +- rename: + field: elasticsearch.slowlog.took_millis + target_field: elasticsearch.slowlog.duration + ignore_missing: true +- grok: + field: elasticsearch.slowlog.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + patterns: + - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? + - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] +- remove: + field: elasticsearch.slowlog.message +- date: + field: elasticsearch.slowlog.timestamp + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json deleted file mode 100644 index e27d3ce0f81..00000000000 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "description": "Pipeline for parsing elasticsearch slow logs in plaintext format.", - "processors": [ - { - "grok": { - "field": "message", - "pattern_definitions": { - "GREEDYMULTILINE": "(.|\n)*", - "INDEXNAME": "[a-zA-Z0-9_.-]*" - }, - "patterns": [ - "\\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\\]\\[%{WORD:log.level}(%{SPACE})\\]\\[%{DATA:elasticsearch.slowlog.logger}\\]%{SPACE}\\[%{DATA:elasticsearch.node.name}\\](%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\]\\[%{NUMBER:elasticsearch.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:elasticsearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:elasticsearch.slowlog.duration:long}\\],)?%{SPACE}(type\\[%{DATA:elasticsearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:elasticsearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:elasticsearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:elasticsearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:elasticsearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:elasticsearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:elasticsearch.slowlog.extra_source}\\])?,?" - ] - } - }, - { - "split": { - "if": "ctx.elasticsearch.slowlog?.stats != ''", - "field": "elasticsearch.slowlog.stats", - "separator": ",", - "ignore_missing": true - } - }, - { - "date": { - "if": "ctx.event.timezone == null", - "field": "elasticsearch.slowlog.timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, - { - "date": { - "if": "ctx.event.timezone != null", - "field": "elasticsearch.slowlog.timestamp", - "target_field": "@timestamp", - "formats": [ - "yyyy-MM-dd'T'HH:mm:ss,SSS" - ], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.yml new file mode 100644 index 00000000000..03a06642740 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.yml @@ -0,0 +1,41 @@ +description: Pipeline for parsing elasticsearch slow logs in plaintext format. +processors: +- grok: + field: message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + patterns: + - \[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\]\[%{WORD:log.level}(%{SPACE})\]\[%{DATA:elasticsearch.slowlog.logger}\]%{SPACE}\[%{DATA:elasticsearch.node.name}\](%{SPACE})?(\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})?(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})?%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? +- split: + if: ctx.elasticsearch.slowlog?.stats != '' + field: elasticsearch.slowlog.stats + separator: ',' + ignore_missing: true +- date: + if: ctx.event.timezone == null + field: elasticsearch.slowlog.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx.event.timezone != null + field: elasticsearch.slowlog.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss,SSS + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json deleted file mode 100644 index e36d0fbf1b0..00000000000 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "description": "Pipeline for parsing elasticsearch slow logs.", - "processors": [ - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "grok": { - "field": "message", - "patterns": [ - "^%{CHAR:first_char}" - ], - "pattern_definitions": { - "CHAR": "." - } - } - }, - { - "pipeline": { - "if": "ctx.first_char != '{'", - "name": "{< IngestPipeline "pipeline-plaintext" >}" - } - }, - { - "pipeline": { - "if": "ctx.first_char == '{'", - "name": "{< IngestPipeline "pipeline-json" >}" - } - }, - { - "remove": { - "field": "elasticsearch.slowlog.timestamp" - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.event.duration = Math.round(ctx.elasticsearch.slowlog.duration * params.scale)", - "params": { - "scale": 1000000 - }, - "if": "ctx.elasticsearch.slowlog?.duration != null" - } - }, - { - "remove": { - "field": "elasticsearch.slowlog.duration", - "ignore_missing": true - } - }, - { - "remove": { - "field": [ - "first_char" - ] - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml new file mode 100644 index 00000000000..f04c42d5bc2 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -0,0 +1,60 @@ +description: Pipeline for parsing elasticsearch slow logs. +processors: +- rename: + field: '@timestamp' + target_field: event.created +- grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . +- pipeline: + if: ctx.first_char != '{' + name: '{< IngestPipeline "pipeline-plaintext" >}' +- pipeline: + if: ctx.first_char == '{' + name: '{< IngestPipeline "pipeline-json" >}' +- remove: + field: elasticsearch.slowlog.timestamp +- script: + lang: painless + source: ctx.event.duration = Math.round(ctx.elasticsearch.slowlog.duration * params.scale) + params: + scale: 1000000 + if: ctx.elasticsearch.slowlog?.duration != null +- remove: + field: elasticsearch.slowlog.duration + ignore_missing: true +- set: + field: event.kind + value: event +- set: + field: event.category + value: database +- script: + lang: painless + source: >- + def errorLevels = ['FATAL', 'ERROR']; + if (ctx?.log?.level != null) { + if (errorLevels.contains(ctx.log.level)) { + ctx.event.type = 'error'; + } else { + ctx.event.type = 'info'; + } + } +- set: + field: host.name + value: "{{elasticsearch.node.name}}" + if: "ctx?.elasticsearch?.node?.name != null" +- set: + field: host.id + value: "{{elasticsearch.node.id}}" + if: "ctx?.elasticsearch?.node?.id != null" +- remove: + field: + - first_char +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/elasticsearch/slowlog/manifest.yml b/filebeat/module/elasticsearch/slowlog/manifest.yml index e17acada052..caddd94158b 100644 --- a/filebeat/module/elasticsearch/slowlog/manifest.yml +++ b/filebeat/module/elasticsearch/slowlog/manifest.yml @@ -19,7 +19,7 @@ var: - c:/ProgramData/Elastic/Elasticsearch/logs/*_index_indexing_slowlog.json ingest_pipeline: - - ingest/pipeline.json - - ingest/pipeline-plaintext.json - - ingest/pipeline-json.json + - ingest/pipeline.yml + - ingest/pipeline-plaintext.yml + - ingest/pipeline-json.yml input: config/slowlog.yml diff --git a/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json index 9f621a0e9d4..e8e26f45164 100644 --- a/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/auditlog_index_indexing_slowlog.log-expected.json @@ -10,10 +10,13 @@ "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"module\":\"system\",\"rtt\":9610,\"name\":\"network\"},\"system\":{\"network\":{\"name\":\"bridg\",\"in\":{\"packets\":0,\"errors\":0,\"dropped\":0,\"bytes\":0},\"out\":{\"errors\":0,\"dropped\":0,\"packets\":1,\"bytes\":342}}},\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", "elasticsearch.slowlog.took": "221micros", "elasticsearch.slowlog.type": "doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -32,10 +35,13 @@ "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"rtt\":9616,\"name\":\"network\",\"module\":\"system\"},\"system\":{\"network\":{\"name\":\"utun0\",\"in\":{\"dropped\":0,\"bytes\":0,\"packets\":0,\"errors\":0},\"out\":{\"packets\":2,\"bytes\":200,\"errors\":0,\"dropped\":0}}},\"beat\":{\"version\":\"6.3.0\",\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", "elasticsearch.slowlog.took": "388.6micros", "elasticsearch.slowlog.type": "doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -54,10 +60,13 @@ "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:40.799Z\",\"metricset\":{\"rtt\":9640,\"name\":\"network\",\"module\":\"system\"},\"system\":{\"network\":{\"name\":\"utun1\",\"in\":{\"packets\":200,\"errors\":0,\"dropped\":0,\"bytes\":44296},\"out\":{\"errors\":0,\"dropped\":0,\"packets\":208,\"bytes\":59626}}},\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", "elasticsearch.slowlog.took": "287.1micros", "elasticsearch.slowlog.type": "doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -75,10 +84,13 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.took": "1.7ms", "elasticsearch.slowlog.type": "doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -97,10 +109,13 @@ "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:42.117Z\",\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"},\"metricset\":{\"module\":\"system\",\"rtt\":39463,\"name\":\"process\"},\"system\":{\"process\":{\"state\":\"running\",\"pid\":6274,\"name\":\"iTerm2\",\"cmdline\":\"/Applications/iTerm.app/Contents/MacOS/iTerm2\",\"ppid\":1,\"pgid\":6274,\"username\":\"rado\",\"memory\":{\"size\":6263349248,\"rss\":{\"bytes\":226975744,\"pct\":0.0132},\"share\":0},\"cpu\":{\"total\":{\"value\":921790,\"pct\":0.1368,\"norm\":{\"pct\":0.0342}},\"start_time\":\"2018-07-02T10:40:29.756Z\"}}}}", "elasticsearch.slowlog.took": "560.6micros", "elasticsearch.slowlog.type": "doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -119,10 +134,13 @@ "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T21:50:42.117Z\",\"beat\":{\"name\":\"Rados-MacBook-Pro.local\",\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"},\"metricset\":{\"name\":\"process\",\"module\":\"system\",\"rtt\":39476},\"system\":{\"process\":{\"username\":\"rado\",\"state\":\"running\",\"cmdline\":\"com.docker.hyperkit -A -u -F vms/0/hyperkit.pid -c 2 -m 6144M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-vpnkit,path=s50,uuid=18fcb277-636a-4fd7-99d2-9bd2dd50a58c -U b1496a26-aed9-4ee1-818d-a3683593b754 -s 2:0,ahci-hd,file:///Users/rado/Library/Containers/com.docker.docker/Data/vms/0/Docker.qcow2?sync=os\\u0026buffered=1,format=qcow,qcow-config=discard=true;compact_after_unmaps=262144;keep_erased=262144;runtime_asserts=false -s 3,virtio-sock,guest_cid=3,path=vms/0,guest_forwards=2376;1525 -s 4,ahci-cd,/Applications/Docker.app/Contents/Resources/linuxkit/docker-for-mac.iso -s 5,ahci-cd,vms/0/config.iso -s 6,virtio-rnd -s 7,virtio-9p,path=s51,tag=port -l com1,autopty=vms/0/tty,log=vms/0/console-ring -f bootrom,/Applications/Docker.app/Contents/Resources/uefi/UEFI.fd,,\",\"ppid\":559,\"pgid\":555,\"name\":\"com.docker.hype\",\"cpu\":{\"total\":{\"pct\":0.1181,\"norm\":{\"pct\":0.0295},\"value\":8.7575e+06},\"start_time\":\"2018-07-01T22:13:07.748Z\"},\"pid\":567,\"memory\":{\"share\":0,\"size\":11128897536,\"rss\":{\"pct\":0.0205,\"bytes\":352854016}}}}}", "elasticsearch.slowlog.took": "469.9micros", "elasticsearch.slowlog.type": "doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", diff --git a/filebeat/module/elasticsearch/slowlog/test/es74_index_indexing_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es74_index_indexing_slowlog-json.log-expected.json index d6c2c575c90..1edcde3b8d9 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es74_index_indexing_slowlog-json.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/es74_index_indexing_slowlog-json.log-expected.json @@ -13,10 +13,14 @@ "elasticsearch.slowlog.source": "{\"foo\":\"bar\"}", "elasticsearch.slowlog.took": "3ms", "elasticsearch.slowlog.types": "t", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 3000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "erxPlzmKQOGUdrDxGrww_g", "input.type": "log", "log.level": "WARN", "log.offset": 0, @@ -36,10 +40,14 @@ "elasticsearch.slowlog.source": "{\"foo\":\"bar\"}", "elasticsearch.slowlog.took": "2.3ms", "elasticsearch.slowlog.types": "_doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 2000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "BgSzU7SUTgeRYqzbiyf1sA", "input.type": "log", "log.level": "WARN", "log.offset": 409, diff --git a/filebeat/module/elasticsearch/slowlog/test/es74_index_search_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es74_index_search_slowlog-json.log-expected.json index c59127f6b77..03d013dfefa 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es74_index_search_slowlog-json.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/es74_index_search_slowlog-json.log-expected.json @@ -16,10 +16,14 @@ "elasticsearch.slowlog.total_hits": "3 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 6000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "gfFy7VKWSXqOOOE1no4s0Q", "input.type": "log", "log.level": "WARN", "log.offset": 0, @@ -43,10 +47,14 @@ "elasticsearch.slowlog.total_hits": "3 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 17000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "gfFy7VKWSXqOOOE1no4s0Q", "input.type": "log", "log.level": "WARN", "log.offset": 837, @@ -70,10 +78,14 @@ "elasticsearch.slowlog.total_hits": "83 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 4000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "gfFy7VKWSXqOOOE1no4s0Q", "input.type": "log", "log.level": "WARN", "log.offset": 4113, @@ -97,10 +109,14 @@ "elasticsearch.slowlog.total_hits": "83 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 5000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "gfFy7VKWSXqOOOE1no4s0Q", "input.type": "log", "log.level": "WARN", "log.offset": 5313, @@ -124,10 +140,14 @@ "elasticsearch.slowlog.total_hits": "83 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 5000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "gfFy7VKWSXqOOOE1no4s0Q", "input.type": "log", "log.level": "WARN", "log.offset": 6516, @@ -151,10 +171,14 @@ "elasticsearch.slowlog.total_hits": "83 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 4000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "gfFy7VKWSXqOOOE1no4s0Q", "input.type": "log", "log.level": "WARN", "log.offset": 7720, @@ -178,10 +202,14 @@ "elasticsearch.slowlog.total_hits": "83 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 9000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "gfFy7VKWSXqOOOE1no4s0Q", "input.type": "log", "log.level": "WARN", "log.offset": 9350, @@ -205,10 +233,14 @@ "elasticsearch.slowlog.total_hits": "83 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 4000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "gfFy7VKWSXqOOOE1no4s0Q", "input.type": "log", "log.level": "WARN", "log.offset": 10564, @@ -230,10 +262,14 @@ "elasticsearch.slowlog.took": "464.2micros", "elasticsearch.slowlog.total_hits": "1 hits", "elasticsearch.slowlog.total_shards": "1", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "BgSzU7SUTgeRYqzbiyf1sA", "input.type": "log", "log.level": "WARN", "log.offset": 11774, diff --git a/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log-expected.json index e0991f52462..456fce2bfb6 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/es_index_indexing_slowlog-json.log-expected.json @@ -13,10 +13,14 @@ "elasticsearch.slowlog.source_query": "{\"somefield\":\"somevalue\"}", "elasticsearch.slowlog.took": "4.6ms", "elasticsearch.slowlog.type": "_doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 4000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "U7rdLkcqR9eRvOiyLmr_qQ", "input.type": "log", "log.level": "WARN", "log.offset": 0, @@ -37,10 +41,14 @@ "elasticsearch.slowlog.source_query": "{\"somefield\":\"somevalue\"}", "elasticsearch.slowlog.took": "803micros", "elasticsearch.slowlog.type": "_doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "U7rdLkcqR9eRvOiyLmr_qQ", "input.type": "log", "log.level": "WARN", "log.offset": 409, diff --git a/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log-expected.json index e7933d2368c..2d1523d3526 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/es_index_search_slowlog-json.log-expected.json @@ -9,10 +9,14 @@ "elasticsearch.node.name": "node-0", "elasticsearch.shard.id": "0", "elasticsearch.slowlog.took": "70.4micros", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "U7rdLkcqR9eRvOiyLmr_qQ", "input.type": "log", "log.level": "WARN", "log.offset": 0, @@ -29,10 +33,14 @@ "elasticsearch.node.name": "node-0", "elasticsearch.shard.id": "0", "elasticsearch.slowlog.took": "731.3micros", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "U7rdLkcqR9eRvOiyLmr_qQ", "input.type": "log", "log.level": "WARN", "log.offset": 429, @@ -49,10 +57,14 @@ "elasticsearch.node.name": "node-0", "elasticsearch.shard.id": "0", "elasticsearch.slowlog.took": "9.9ms", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 9000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "U7rdLkcqR9eRvOiyLmr_qQ", "input.type": "log", "log.level": "WARN", "log.offset": 859, diff --git a/filebeat/module/elasticsearch/slowlog/test/slowlogs-json.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/slowlogs-json.log-expected.json index 3de1770efc9..15513533ed9 100644 --- a/filebeat/module/elasticsearch/slowlog/test/slowlogs-json.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/slowlogs-json.log-expected.json @@ -15,10 +15,14 @@ "elasticsearch.slowlog.total_hits": "1 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "[]", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 9000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "rR5HxA67QeeR7VOuiu64Lg", "input.type": "log", "log.level": "WARN", "log.offset": 0, @@ -41,10 +45,14 @@ "elasticsearch.slowlog.total_hits": "1 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "[]", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "rR5HxA67QeeR7VOuiu64Lg", "input.type": "log", "log.level": "WARN", "log.offset": 550, @@ -67,10 +75,14 @@ "elasticsearch.slowlog.total_hits": "1 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "[]", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "rR5HxA67QeeR7VOuiu64Lg", "input.type": "log", "log.level": "WARN", "log.offset": 1082, @@ -94,10 +106,14 @@ "elasticsearch.slowlog.total_hits": "0 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "[\"type1\", \"type2\"]", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 2000000, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "rR5HxA67QeeR7VOuiu64Lg", "input.type": "log", "log.level": "WARN", "log.offset": 1581, @@ -121,10 +137,14 @@ "elasticsearch.slowlog.total_hits": "0 hits", "elasticsearch.slowlog.total_shards": "1", "elasticsearch.slowlog.types": "[\"type1\"]", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 0, + "event.kind": "event", "event.module": "elasticsearch", + "event.type": "info", "fileset.name": "slowlog", + "host.id": "rR5HxA67QeeR7VOuiu64Lg", "input.type": "log", "log.flags": [ "multiline" diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index 55fb7a6c3b6..2f2cfbf55d5 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -15,10 +15,13 @@ "elasticsearch.slowlog.total_hits": 19435, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 4000000, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -39,10 +42,13 @@ "elasticsearch.slowlog.total_hits": 19435, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 10000000, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -63,10 +69,13 @@ "elasticsearch.slowlog.total_hits": 0, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 124000000, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -87,10 +96,13 @@ "elasticsearch.slowlog.total_hits": 0, "elasticsearch.slowlog.total_shards": 1, "elasticsearch.slowlog.types": "", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 7000000, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -109,10 +121,13 @@ "elasticsearch.slowlog.source_query": "{\"@timestamp\":\"2018-07-04T13:47:50.747Z\",\"system\":{\"process\":{\"ppid\":34526,\"state\":\"running\",\"cpu\":{\"total\":{\"value\":734879,\"pct\":0.0173,\"norm\":{\"pct\":0.0043}},\"start_time\":\"2018-07-04T06:56:34.863Z\"},\"pgid\":34526,\"cmdline\":\"/Applications/Firefox.app/Contents/MacOS/plugin-container.app/Contents/MacOS/plugin-container -childID 1 -isForBrowser -prefsLen 22119 -schedulerPrefs 0001,2 -greomni /Applications/Firefox.app/Contents/Resources/omni.ja -appomni /Applications/Firefox.app/Contents/Resources/browser/omni.ja -appdir /Applications/Firefox.app/Contents/Resources/browser -profile /Users/rado/Library/Application Support/Firefox/Profiles/pt6eoq1j.default-1484133908360 34526 gecko-crash-server-pipe.34526 org.mozilla.machname.231926932 tab\",\"name\":\"plugin-containe\",\"memory\":{\"size\":7489249280,\"rss\":{\"bytes\":567619584,\"pct\":0.033},\"share\":0},\"pid\":34528,\"username\":\"rado\"}},\"metricset\":{\"name\":\"process\",\"module\":\"system\",\"rtt\":43856},\"beat\":{\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\",\"name\":\"Rados-MacBook-Pro.local\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}", "elasticsearch.slowlog.took": "1.4ms", "elasticsearch.slowlog.type": "doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -130,10 +145,13 @@ "elasticsearch.slowlog.routing": "", "elasticsearch.slowlog.took": "1.7ms", "elasticsearch.slowlog.type": "doc", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 1000000, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", @@ -153,10 +171,13 @@ "elasticsearch.slowlog.took": "516.4ms", "elasticsearch.slowlog.total_shards": 10, "elasticsearch.slowlog.types": "encounter", + "event.category": "database", "event.dataset": "elasticsearch.slowlog", "event.duration": 516000000, + "event.kind": "event", "event.module": "elasticsearch", "event.timezone": "-02:00", + "event.type": "info", "fileset.name": "slowlog", "input.type": "log", "log.level": "TRACE",