From 2dc56d36c4d7692635dfa4c61f2e44a6867881d1 Mon Sep 17 00:00:00 2001 From: ruflin Date: Tue, 11 Apr 2017 09:53:05 +0200 Subject: [PATCH] Change error to error.message Currently error message in metricbeat are written into the `error` field. In heartbeat `error.message` is used. To unify the error structure and allow in the future to support things like `error.code` and others inside the error, `error` is changed to `error.message`. Currently internally all `error` are just written to `error.message`. In future PR's an internal logic should be added to support `error.code` and others so Metricsets, Prospectors etc. can make use of it. Changes * Change `error` to `error.message` * Move error mappings from heartbeat to libbeat Closes https://github.com/elastic/beats/issues/3951 --- CHANGELOG.asciidoc | 3 +- filebeat/docs/fields.asciidoc | 31 +++++++++++ filebeat/docs/modules-dev-guide.asciidoc | 10 ++-- .../module/apache2/access/ingest/default.json | 2 +- .../module/apache2/error/ingest/pipeline.json | 2 +- .../module/auditd/log/ingest/pipeline.json | 2 +- .../module/mysql/error/ingest/pipeline.json | 2 +- .../module/mysql/slowlog/ingest/pipeline.json | 2 +- .../module/nginx/access/ingest/default.json | 2 +- .../module/nginx/error/ingest/pipeline.json | 2 +- .../module/system/auth/ingest/pipeline.json | 2 +- .../module/system/syslog/ingest/pipeline.json | 2 +- .../module/fileset/ingest/pipeline.json | 2 +- heartbeat/_meta/fields.yml | 14 ----- heartbeat/docs/fields.asciidoc | 54 +++++++++++-------- libbeat/_meta/fields.common.yml | 17 ++++++ metricbeat/docs/fields.asciidoc | 31 +++++++++++ metricbeat/docs/how-metricbeat-works.asciidoc | 4 +- metricbeat/mb/module/event.go | 4 +- metricbeat/mb/module/event_test.go | 2 +- packetbeat/docs/fields.asciidoc | 31 +++++++++++ winlogbeat/docs/fields.asciidoc | 31 +++++++++++ 22 files changed, 196 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 72096a492c2..b2a0f0eb8bf 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -18,7 +18,8 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff] - Configuration files must be owned by the user running the beat or by root, and they must not be writable by others. {pull}3544[3544] {pull}3689[3689] - Usage of field `_type` is now ignored and hardcoded to `doc`. {pull}3757[3757] -- Change vendor manager from glide to govendor {pull}3851[3851] +- Change vendor manager from glide to govendor. {pull}3851[3851] +- Rename `error` field to `error.message`. {pull}3987[3987] *Filebeat* - Always use absolute path for event and registry. This can lead to issues when relative paths were used before. {pull}3328[3328] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index f7a969a6623..7f8973221fd 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -493,6 +493,37 @@ type: object Contains user configurable fields. +[float] +== error Fields + +Error fields containing additional info in case of errors. + + + +[float] +=== error.message + +type: text + +Error message. + + +[float] +=== error.code + +type: long + +Error code. + + +[float] +=== error.type + +type: keyword + +Error type. + + [[exported-fields-cloud]] == Cloud Provider Metadata Fields diff --git a/filebeat/docs/modules-dev-guide.asciidoc b/filebeat/docs/modules-dev-guide.asciidoc index abe17e26841..0db5ec6b8b2 100644 --- a/filebeat/docs/modules-dev-guide.asciidoc +++ b/filebeat/docs/modules-dev-guide.asciidoc @@ -159,7 +159,7 @@ exclude_files: [".gz$"] You'll find this example in the template file that gets generated automatically when you run `make create-fileset`. In this example, the `paths` variable is -used to construct the `paths` list for the <> option. +used to construct the `paths` list for the <> option. Any template files that you add to the `config/` folder need to generate a valid Filebeat prospector configuration in YAML format. The options accepted by the @@ -167,7 +167,7 @@ prospector configuration are documented in this <>. The template files use the templating language defined by the -https://golang.org/pkg/text/template/[Golang standard library]. +https://golang.org/pkg/text/template/[Golang standard library]. Here is another example that also configures multiline stitching: @@ -192,7 +192,7 @@ variables to dynamically switch between configurations. [float] ==== ingest/*.json -The `ingest/` folder contains Elasticsearch +The `ingest/` folder contains Elasticsearch {elasticsearch}/ingest.html[Ingest Node] pipeline configurations. The Ingest Node pipelines are responsible for parsing the log lines and doing other manipulations on the data. @@ -212,7 +212,7 @@ The generator creates a JSON object similar to this one: ], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] @@ -221,7 +221,7 @@ The generator creates a JSON object similar to this one: From here, you would typically add processors to the `processors` array to do the actual parsing. For details on how to use ingest node processors, see the -{elasticsearch}/ingest-processors.html[ingest node documentation]. In +{elasticsearch}/ingest-processors.html[ingest node documentation]. In particular, you will likely find the {elasticsearch}/grok-processor.html[Grok processor] to be useful for parsing. Here is an example for parsing the Nginx access logs. diff --git a/filebeat/module/apache2/access/ingest/default.json b/filebeat/module/apache2/access/ingest/default.json index b62f423968c..69de2b0829b 100644 --- a/filebeat/module/apache2/access/ingest/default.json +++ b/filebeat/module/apache2/access/ingest/default.json @@ -47,7 +47,7 @@ }], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/filebeat/module/apache2/error/ingest/pipeline.json b/filebeat/module/apache2/error/ingest/pipeline.json index ac4e9a9cdcb..417d9890397 100644 --- a/filebeat/module/apache2/error/ingest/pipeline.json +++ b/filebeat/module/apache2/error/ingest/pipeline.json @@ -43,7 +43,7 @@ ], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/filebeat/module/auditd/log/ingest/pipeline.json b/filebeat/module/auditd/log/ingest/pipeline.json index 46dacd82b8a..e2a97600389 100644 --- a/filebeat/module/auditd/log/ingest/pipeline.json +++ b/filebeat/module/auditd/log/ingest/pipeline.json @@ -93,7 +93,7 @@ "on_failure": [ { "set": { - "field": "error", + "field": "error.message", "value": "{{ _ingest.on_failure_message }}" } } diff --git a/filebeat/module/mysql/error/ingest/pipeline.json b/filebeat/module/mysql/error/ingest/pipeline.json index 201cb8fd21f..911302fde57 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.json +++ b/filebeat/module/mysql/error/ingest/pipeline.json @@ -43,7 +43,7 @@ }], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index f943ec68001..e349de4a7c4 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -32,7 +32,7 @@ }], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index cf0441d5126..5f57757267f 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -44,7 +44,7 @@ }], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json index e60aaddeb2e..f8231e04607 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.json +++ b/filebeat/module/nginx/error/ingest/pipeline.json @@ -30,7 +30,7 @@ }], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json index 56ea495bfed..8da95c08960 100644 --- a/filebeat/module/system/auth/ingest/pipeline.json +++ b/filebeat/module/system/auth/ingest/pipeline.json @@ -45,7 +45,7 @@ ], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json index 121905091e9..8555e697169 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.json +++ b/filebeat/module/system/syslog/ingest/pipeline.json @@ -33,7 +33,7 @@ ], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/filebeat/scripts/module/fileset/ingest/pipeline.json b/filebeat/scripts/module/fileset/ingest/pipeline.json index 90fde13d235..ad3fb2ec868 100644 --- a/filebeat/scripts/module/fileset/ingest/pipeline.json +++ b/filebeat/scripts/module/fileset/ingest/pipeline.json @@ -4,7 +4,7 @@ ], "on_failure" : [{ "set" : { - "field" : "error", + "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } }] diff --git a/heartbeat/_meta/fields.yml b/heartbeat/_meta/fields.yml index 6b416ab4941..536927272b3 100644 --- a/heartbeat/_meta/fields.yml +++ b/heartbeat/_meta/fields.yml @@ -130,17 +130,3 @@ description: > Boolean indicator if monitor could validate the service to be available. - - name: error - type: group - description: > - Reason monitor flagging a service as down. - fields: - - name: type - type: keyword - description: > - Failure type. For example `io` or `validate`. - - - name: message - type: text - description: > - Failure description. diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index 58b338ca337..a8c71307e11 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -76,6 +76,37 @@ type: object Contains user configurable fields. +[float] +== error Fields + +Error fields containing additional info in case of errors. + + + +[float] +=== error.message + +type: text + +Error message. + + +[float] +=== error.code + +type: long + +Error code. + + +[float] +=== error.type + +type: keyword + +Error type. + + [[exported-fields-cloud]] == Cloud Provider Metadata Fields @@ -325,26 +356,3 @@ required: True Boolean indicator if monitor could validate the service to be available. -[float] -== error Fields - -Reason monitor flagging a service as down. - - - -[float] -=== error.type - -type: keyword - -Failure type. For example `io` or `validate`. - - -[float] -=== error.message - -type: text - -Failure description. - - diff --git a/libbeat/_meta/fields.common.yml b/libbeat/_meta/fields.common.yml index 0992f0ce645..de6563a9d2e 100644 --- a/libbeat/_meta/fields.common.yml +++ b/libbeat/_meta/fields.common.yml @@ -42,3 +42,20 @@ description: > Contains user configurable fields. + - name: error + type: group + description: > + Error fields containing additional info in case of errors. + fields: + - name: message + type: text + description: > + Error message. + - name: code + type: long + description: > + Error code. + - name: type + type: keyword + description: > + Error type. diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4a52575a63a..a7cca35d717 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -426,6 +426,37 @@ type: object Contains user configurable fields. +[float] +== error Fields + +Error fields containing additional info in case of errors. + + + +[float] +=== error.message + +type: text + +Error message. + + +[float] +=== error.code + +type: long + +Error code. + + +[float] +=== error.type + +type: keyword + +Error type. + + [[exported-fields-ceph]] == ceph Fields diff --git a/metricbeat/docs/how-metricbeat-works.asciidoc b/metricbeat/docs/how-metricbeat-works.asciidoc index d52c30ccaf1..667bb589a05 100644 --- a/metricbeat/docs/how-metricbeat-works.asciidoc +++ b/metricbeat/docs/how-metricbeat-works.asciidoc @@ -98,7 +98,9 @@ reachable: "hostname": "host.example.com", "name": "host.example.com" }, - "error": "Get http://127.0.0.1/server-status?auto: dial tcp 127.0.0.1:80: getsockopt: connection refused", + "error": { + "message": "Get http://127.0.0.1/server-status?auto: dial tcp 127.0.0.1:80: getsockopt: connection refused", + }, "metricset": { "module": "apache", "name": "status", diff --git a/metricbeat/mb/module/event.go b/metricbeat/mb/module/event.go index d2d28be43d7..cefe684a064 100644 --- a/metricbeat/mb/module/event.go +++ b/metricbeat/mb/module/event.go @@ -103,7 +103,9 @@ func (b EventBuilder) Build() (common.MapStr, error) { // Adds error to event in case error happened if b.fetchErr != nil { - event["error"] = b.fetchErr.Error() + event["error"] = common.MapStr{ + "message": b.fetchErr.Error(), + } } return event, nil diff --git a/metricbeat/mb/module/event_test.go b/metricbeat/mb/module/event_test.go index 20f77148044..4fa819db3fe 100644 --- a/metricbeat/mb/module/event_test.go +++ b/metricbeat/mb/module/event_test.go @@ -65,7 +65,7 @@ func TestEventBuilderError(t *testing.T) { t.Fatal(err) } - assert.Equal(t, errFetch.Error(), event["error"]) + assert.Equal(t, errFetch.Error(), event["error"].(common.MapStr)["message"]) } func TestEventBuilderNoHost(t *testing.T) { diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index e553e41afd3..5e3191b73f8 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -415,6 +415,37 @@ type: object Contains user configurable fields. +[float] +== error Fields + +Error fields containing additional info in case of errors. + + + +[float] +=== error.message + +type: text + +Error message. + + +[float] +=== error.code + +type: long + +Error code. + + +[float] +=== error.type + +type: keyword + +Error type. + + [[exported-fields-cassandra]] == Cassandra Fields diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc index 7085b9e7512..842ea03e08a 100644 --- a/winlogbeat/docs/fields.asciidoc +++ b/winlogbeat/docs/fields.asciidoc @@ -77,6 +77,37 @@ type: object Contains user configurable fields. +[float] +== error Fields + +Error fields containing additional info in case of errors. + + + +[float] +=== error.message + +type: text + +Error message. + + +[float] +=== error.code + +type: long + +Error code. + + +[float] +=== error.type + +type: keyword + +Error type. + + [[exported-fields-cloud]] == Cloud Provider Metadata Fields