From 9afff8fb8aedf80eae34283feeb2421163f6f752 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 29 Nov 2018 14:06:12 -0500 Subject: [PATCH 1/6] Map 2 Icinga module fields to ECS: - icinga.debug.message => message - icinga.debug.severity => log.level --- .../module/icinga/debug/ingest/pipeline.json | 7 +------ .../icinga/debug/test/test.log-expected.json | 18 +++++++++--------- .../module/icinga/main/ingest/pipeline.json | 7 +------ .../icinga/main/test/test.log-expected.json | 18 +++++++++--------- .../module/icinga/startup/ingest/pipeline.json | 7 +------ .../icinga/startup/test/test.log-expected.json | 12 ++++++------ 6 files changed, 27 insertions(+), 42 deletions(-) diff --git a/filebeat/module/icinga/debug/ingest/pipeline.json b/filebeat/module/icinga/debug/ingest/pipeline.json index 4f94e65d7472..65abfffca9f8 100644 --- a/filebeat/module/icinga/debug/ingest/pipeline.json +++ b/filebeat/module/icinga/debug/ingest/pipeline.json @@ -4,7 +4,7 @@ "grok": { "field": "message", "patterns":[ - "\\[%{TIMESTAMP:icinga.debug.timestamp}\\] %{WORD:icinga.debug.severity}/%{WORD:icinga.debug.facility}: %{GREEDYMULTILINE:icinga.debug.message}" + "\\[%{TIMESTAMP:icinga.debug.timestamp}\\] %{WORD:log.level}/%{WORD:icinga.debug.facility}: %{GREEDYMULTILINE:message}" ], "ignore_missing": true, "pattern_definitions": { @@ -13,11 +13,6 @@ } } }, - { - "remove": { - "field": "message" - } - }, { "date": { "field": "icinga.debug.timestamp", diff --git a/filebeat/module/icinga/debug/test/test.log-expected.json b/filebeat/module/icinga/debug/test/test.log-expected.json index d412a561741e..84cc45549e66 100644 --- a/filebeat/module/icinga/debug/test/test.log-expected.json +++ b/filebeat/module/icinga/debug/test/test.log-expected.json @@ -4,29 +4,29 @@ "event.dataset": "debug", "event.module": "icinga", "icinga.debug.facility": "GraphiteWriter", - "icinga.debug.message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", - "icinga.debug.severity": "debug", "input.type": "log", - "log.offset": 0 + "log.level": "debug", + "log.offset": 0, + "message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'." }, { "@timestamp": "2017-04-04T11:43:09.000Z", "event.dataset": "debug", "event.module": "icinga", "icinga.debug.facility": "IdoMysqlConnection", - "icinga.debug.message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", - "icinga.debug.severity": "debug", "input.type": "log", - "log.offset": 141 + "log.level": "debug", + "log.offset": 141, + "message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333" }, { "@timestamp": "2017-04-04T11:43:11.000Z", "event.dataset": "debug", "event.module": "icinga", "icinga.debug.facility": "Process", - "icinga.debug.message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", - "icinga.debug.severity": "notice", "input.type": "log", - "log.offset": 1763 + "log.level": "notice", + "log.offset": 1763, + "message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288" } ] \ No newline at end of file diff --git a/filebeat/module/icinga/main/ingest/pipeline.json b/filebeat/module/icinga/main/ingest/pipeline.json index ee4a0e76660b..b11b0cabfbfd 100644 --- a/filebeat/module/icinga/main/ingest/pipeline.json +++ b/filebeat/module/icinga/main/ingest/pipeline.json @@ -4,7 +4,7 @@ "grok": { "field": "message", "patterns":[ - "\\[%{TIMESTAMP:icinga.main.timestamp}\\] %{WORD:icinga.main.severity}/%{WORD:icinga.main.facility}: %{GREEDYMULTILINE:icinga.main.message}" + "\\[%{TIMESTAMP:icinga.main.timestamp}\\] %{WORD:log.level}/%{WORD:icinga.main.facility}: %{GREEDYMULTILINE:message}" ], "ignore_missing": true, "pattern_definitions": { @@ -13,11 +13,6 @@ } } }, - { - "remove": { - "field": "message" - } - }, { "date": { "field": "icinga.main.timestamp", diff --git a/filebeat/module/icinga/main/test/test.log-expected.json b/filebeat/module/icinga/main/test/test.log-expected.json index cc324c7ec14b..b77351798561 100644 --- a/filebeat/module/icinga/main/test/test.log-expected.json +++ b/filebeat/module/icinga/main/test/test.log-expected.json @@ -4,32 +4,32 @@ "event.dataset": "main", "event.module": "icinga", "icinga.main.facility": "Notification", - "icinga.main.message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", - "icinga.main.severity": "information", "input.type": "log", - "log.offset": 0 + "log.level": "information", + "log.offset": 0, + "message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'" }, { "@timestamp": "2017-04-04T09:16:34.000Z", "event.dataset": "main", "event.module": "icinga", "icinga.main.facility": "PluginNotificationTask", - "icinga.main.message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", - "icinga.main.severity": "warning", "input.type": "log", "log.flags": [ "multiline" ], - "log.offset": 133 + "log.level": "warning", + "log.offset": 133, + "message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n" }, { "@timestamp": "2017-04-04T09:16:48.000Z", "event.dataset": "main", "event.module": "icinga", "icinga.main.facility": "IdoMysqlConnection", - "icinga.main.message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", - "icinga.main.severity": "information", "input.type": "log", - "log.offset": 518 + "log.level": "information", + "log.offset": 518, + "message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);" } ] \ No newline at end of file diff --git a/filebeat/module/icinga/startup/ingest/pipeline.json b/filebeat/module/icinga/startup/ingest/pipeline.json index b140c6368b1f..9528010c7168 100644 --- a/filebeat/module/icinga/startup/ingest/pipeline.json +++ b/filebeat/module/icinga/startup/ingest/pipeline.json @@ -4,18 +4,13 @@ "grok": { "field": "message", "patterns":[ - "%{WORD:icinga.startup.severity}/%{WORD:icinga.startup.facility}: %{GREEDYMULTILINE:icinga.startup.message}" + "%{WORD:log.level}/%{WORD:icinga.startup.facility}: %{GREEDYMULTILINE:message}" ], "ignore_missing": true, "pattern_definitions": { "GREEDYMULTILINE": "(.|\n)*" } } - }, - { - "remove": { - "field": "message" - } }], "on_failure" : [{ "set" : { diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json index f793fea6f9c5..1bd665dcf4a3 100644 --- a/filebeat/module/icinga/startup/test/test.log-expected.json +++ b/filebeat/module/icinga/startup/test/test.log-expected.json @@ -3,18 +3,18 @@ "event.dataset": "startup", "event.module": "icinga", "icinga.startup.facility": "cli", - "icinga.startup.message": "Icinga application loader (version: r2.6.3-1)", - "icinga.startup.severity": "information", "input.type": "log", - "log.offset": 0 + "log.level": "information", + "log.offset": 0, + "message": "Icinga application loader (version: r2.6.3-1)" }, { "event.dataset": "startup", "event.module": "icinga", "icinga.startup.facility": "cli", - "icinga.startup.message": "Loading configuration file(s).", - "icinga.startup.severity": "information", "input.type": "log", - "log.offset": 63 + "log.level": "information", + "log.offset": 63, + "message": "Loading configuration file(s)." } ] \ No newline at end of file From db9ab6bcafa7c58eb730c194e1f07c40aa7a4b3f Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 11:48:49 -0500 Subject: [PATCH 2/6] Clean up the comments a bit in `ecs-migration.yml`. --- dev-tools/ecs-migration.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 5303b59b7a7f..09b740cec5ec 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -97,6 +97,8 @@ alias6: true +# Filebeat modules + # Suricata module - from: source_ecs.ip @@ -131,6 +133,8 @@ to: source.geo.region_iso_code alias: true +## System module + - from: system.syslog.hostname to: host.hostname alias: true @@ -188,8 +192,6 @@ to: source.geo.* alias: true -# Filebeat modules - ## Apache module - from: apache2.access.user_name @@ -254,7 +256,7 @@ to: process.thread.id alias: true -# IIS module +## IIS module - from: iis.access.server_ip to: destination.ip @@ -312,8 +314,6 @@ to: source.geo.region_iso_code alias: true -# Note: `http` is not officially in ECS yet - - from: iis.access.method to: http.request.method alias: true @@ -326,7 +326,8 @@ to: http.request.referrer alias: true -# HAProxy module +## HAProxy module + - from: haproxy.client.port to: source.port alias: true @@ -375,6 +376,8 @@ to: network.forwarded_ip alias: true +## NGINX module + - from: nginx.access.user_name to: user.name alias: true @@ -387,8 +390,6 @@ to: user_agent.original alias: true -# Note: `http` is not officially in ECS yet - - from: nginx.access.response_code to: http.response.status_code alias: true @@ -447,12 +448,16 @@ to: message alias: true -# From Auditbeat's auditd module. +# Auditbeat + +## From Auditbeat's auditd module. - from: source.hostname to: source.domain alias: true -# Metricbeat base fields +# Metricbeat + +## Metricbeat base fields - from: metricset.name to: event.dataset alias: false @@ -477,4 +482,3 @@ to: event.dataset alias: false comment: No alias mapping as field did not always exist - From e0d3bd6482a8c83f11037fe70e836131bcf44e90 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 11:52:38 -0500 Subject: [PATCH 3/6] Document icinga field moves in ecs-migration --- dev-tools/ecs-migration.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 09b740cec5ec..41243ad1a557 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -448,6 +448,29 @@ to: message alias: true +## Icinga module + +- from: icinga.debug.message + to: message + alias: true +- from: icinga.debug.severity + to: message + alias: true + +- from: icinga.main.message + to: message + alias: true +- from: icinga.main.severity + to: message + alias: true + +- from: icinga.startup.message + to: message + alias: true +- from: icinga.startup.severity + to: message + alias: true + # Auditbeat ## From Auditbeat's auditd module. From aeabd4d18e6a0f81dda0b90e5a5a475190680af4 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 11:57:52 -0500 Subject: [PATCH 4/6] Alias migrated Icinga fields, including the new `migration` attribute --- filebeat/docs/fields.asciidoc | 30 ++++++++----------- filebeat/module/icinga/debug/_meta/fields.yml | 14 ++++----- filebeat/module/icinga/fields.go | 2 +- filebeat/module/icinga/main/_meta/fields.yml | 14 ++++----- .../module/icinga/startup/_meta/fields.yml | 14 ++++----- 5 files changed, 34 insertions(+), 40 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 88fde061b9a4..f116680db01d 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -4813,20 +4813,18 @@ Specifies what component of Icinga logged the message. *`icinga.debug.severity`*:: + -- -type: keyword - -Possible values are "debug", "notice", "information", "warning" or "critical". +type: alias +alias to: log.level -- *`icinga.debug.message`*:: + -- -type: text - -The logged message. +type: alias +alias to: message -- @@ -4850,20 +4848,18 @@ Specifies what component of Icinga logged the message. *`icinga.main.severity`*:: + -- -type: keyword - -Possible values are "debug", "notice", "information", "warning" or "critical". +type: alias +alias to: log.level -- *`icinga.main.message`*:: + -- -type: text - -The logged message. +type: alias +alias to: message -- @@ -4887,20 +4883,18 @@ Specifies what component of Icinga logged the message. *`icinga.startup.severity`*:: + -- -type: keyword - -Possible values are "debug", "notice", "information", "warning" or "critical". +type: alias +alias to: log.level -- *`icinga.startup.message`*:: + -- -type: text - -The logged message. +type: alias +alias to: message -- diff --git a/filebeat/module/icinga/debug/_meta/fields.yml b/filebeat/module/icinga/debug/_meta/fields.yml index 383141f7755d..16a398cb008c 100644 --- a/filebeat/module/icinga/debug/_meta/fields.yml +++ b/filebeat/module/icinga/debug/_meta/fields.yml @@ -7,12 +7,12 @@ type: keyword description: > Specifies what component of Icinga logged the message. + - name: severity - type: keyword - description: > - Possible values are "debug", "notice", "information", "warning" or - "critical". + type: alias + path: log.level + migration: true - name: message - type: text - description: > - The logged message. + type: alias + path: message + migration: true diff --git a/filebeat/module/icinga/fields.go b/filebeat/module/icinga/fields.go index 29cc742c5747..1e94862f1086 100644 --- a/filebeat/module/icinga/fields.go +++ b/filebeat/module/icinga/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzskjGO4zAMRXuf4kP1Jgdwsc1WWyywwO4FFJlWiMiiIdHJ+PYDO3bgJJ4BBshUCSubhB7/J/4GB+pLsOPobQEoa6AS5vfYMAVQUXaJW2WJJX4WAHAe4o9UXaACqJlClctxtkG0DS2IQ2nfUgmfpGunzgr1mrNkVbTr/KW7hvsQea5fEtVyzNMG1JKge5qNjHwE8Xm7eHarZqmoto4Da381nIUdqD9Jqm5mn8gb6l9LjmumjNPeKpw0rUSKCqlnmUG8p2oU3lDO1tN2VVymI6WHivsrOfMuEI42dJRhE8GMVzM/YKIoOxq+ONaSGjtght+TTZGjN5B0hzQusbKzwaybmByuelB6068Z+L+n+Xx3p7tstBy/MWQD/pWxJ89YVpv0KlCPjtm04ZW0J0vaewAAAP//F3AQNA==" + return "eJzsksFq8zAQhO9+iiH35AF0+C//qYee+gRba60skSUhrRP89sWOkzrGKQTaQiF7nEEzn2C2OHBvILUERxWgop4NNi+jsKkAy6XOklRiMPhXAcDZxGu0necKaIS9LWb0tgjU8ixxOO0TG7gcuzQpK6m3OfMsy++du6prcXcjz/c/BiUJZWpAEzN0z5ePjPnw0ZXd7NmSZk7UUC1etL8xL2AH7k8x24X3Bd5wb4lraYQLTntS1LFNMXBQxOaC6aNzbEfwlkshx5+4q5SFj5zvUZIXKgsnke7N0LPzfGS/cFtxmc78mjterZzAHmxcf3Wv79pFEn5wFUP8cxR/bRRFKevNAr57F1PDcxoPTmP369v4CAAA///Y9smg" } diff --git a/filebeat/module/icinga/main/_meta/fields.yml b/filebeat/module/icinga/main/_meta/fields.yml index 3eef5defc2f4..3cb1da4d359e 100644 --- a/filebeat/module/icinga/main/_meta/fields.yml +++ b/filebeat/module/icinga/main/_meta/fields.yml @@ -7,12 +7,12 @@ type: keyword description: > Specifies what component of Icinga logged the message. + - name: severity - type: keyword - description: > - Possible values are "debug", "notice", "information", "warning" or - "critical". + type: alias + path: log.level + migration: true - name: message - type: text - description: > - The logged message. + type: alias + path: message + migration: true diff --git a/filebeat/module/icinga/startup/_meta/fields.yml b/filebeat/module/icinga/startup/_meta/fields.yml index 1758cf74bec8..5a3d1e89f52c 100644 --- a/filebeat/module/icinga/startup/_meta/fields.yml +++ b/filebeat/module/icinga/startup/_meta/fields.yml @@ -7,12 +7,12 @@ type: keyword description: > Specifies what component of Icinga logged the message. + - name: severity - type: keyword - description: > - Possible values are "debug", "notice", "information", "warning" or - "critical". + type: alias + path: log.level + migration: true. - name: message - type: text - description: > - The logged message. + type: alias + path: message + migration: true From ee106ea28bb28eb7c3bad81e833a6e40567144d0 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 12:05:51 -0500 Subject: [PATCH 5/6] Changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c8b17492fc9e..46762b4dcc3d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -180,6 +180,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the - Fix saved objects in filebeat haproxy dashboard. {pull}9417[9417] - Use `log.source.address` instead of `log.source.ip` for network input sources. {pull}9487[9487] - Rename many `redis.log.*` fields to map to ECS. {pull}9315[9315] +- Rename many `icinga.*` fields to map to ECS. {pull}9294[9294] *Metricbeat* From e8eb3ddd346635f960ac74c927f1489a16ccad5a Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 09:11:13 -0500 Subject: [PATCH 6/6] Fix the icinga.*.severity field aliases in ecs-migration.yml --- dev-tools/ecs-migration.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 41243ad1a557..f95542a780d8 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -454,21 +454,21 @@ to: message alias: true - from: icinga.debug.severity - to: message + to: log.level alias: true - from: icinga.main.message to: message alias: true - from: icinga.main.severity - to: message + to: log.level alias: true - from: icinga.startup.message to: message alias: true - from: icinga.startup.severity - to: message + to: log.level alias: true # Auditbeat