Skip to content

Commit

Permalink
Improve ECS categorization field mapping in icinga (elastic#16533)
Browse files Browse the repository at this point in the history
- event.kind
- event.type

Closes elastic#16164

(cherry picked from commit b82a427)
  • Loading branch information
leehinman committed Mar 11, 2020
1 parent 3078ed1 commit 1051523
Show file tree
Hide file tree
Showing 13 changed files with 127 additions and 96 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Allow users to override pipeline ID in fileset input config. {issue}9531[9531] {pull}16561[16561]
- Improve ECS categorization field mappings in logstash module. {issue}16169[16169] {pull}16668[16668]
- Improve ECS categorization field mappings in iis module. {issue}16165[16165] {pull}16618[16618]
- Improve ECS categorization field mapping in icinga module. {issue}16164[16164] {pull}16533[16533]

*Heartbeat*

Expand Down
36 changes: 0 additions & 36 deletions filebeat/module/icinga/debug/ingest/pipeline.json

This file was deleted.

39 changes: 39 additions & 0 deletions filebeat/module/icinga/debug/ingest/pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
description: Pipeline for parsing icinga debug logs
processors:
- grok:
field: message
patterns:
- '\[%{TIMESTAMP:icinga.debug.timestamp}\] %{WORD:log.level}/%{WORD:icinga.debug.facility}:
%{GREEDYMULTILINE:message}'
ignore_missing: true
pattern_definitions:
TIMESTAMP: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}'
GREEDYMULTILINE: |-
(.|
)*
- date:
field: icinga.debug.timestamp
target_field: '@timestamp'
formats:
- yyyy-MM-dd HH:mm:ss Z
ignore_failure: true
- remove:
field: icinga.debug.timestamp
- set:
field: event.kind
value: event
- script:
lang: painless
source: >-
def errorLevels = ["warning", "critical"];
if (ctx?.log?.level != null) {
if (errorLevels.contains(ctx.log.level)) {
ctx.event.type = "error";
} else {
ctx.event.type = "info";
}
}
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
2 changes: 1 addition & 1 deletion filebeat/module/icinga/debug/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ var:
os.windows:
- c:/programdata/icinga2/var/log/icinga2/debug.log*

ingest_pipeline: ingest/pipeline.json
ingest_pipeline: ingest/pipeline.yml
input: config/debug.yml
6 changes: 6 additions & 0 deletions filebeat/module/icinga/debug/test/test.log-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
{
"@timestamp": "2017-04-04T11:43:09.000Z",
"event.dataset": "icinga.debug",
"event.kind": "event",
"event.module": "icinga",
"event.type": "info",
"fileset.name": "debug",
"icinga.debug.facility": "GraphiteWriter",
"input.type": "log",
Expand All @@ -14,7 +16,9 @@
{
"@timestamp": "2017-04-04T11:43:09.000Z",
"event.dataset": "icinga.debug",
"event.kind": "event",
"event.module": "icinga",
"event.type": "info",
"fileset.name": "debug",
"icinga.debug.facility": "IdoMysqlConnection",
"input.type": "log",
Expand All @@ -26,7 +30,9 @@
{
"@timestamp": "2017-04-04T11:43:11.000Z",
"event.dataset": "icinga.debug",
"event.kind": "event",
"event.module": "icinga",
"event.type": "info",
"fileset.name": "debug",
"icinga.debug.facility": "Process",
"input.type": "log",
Expand Down
36 changes: 0 additions & 36 deletions filebeat/module/icinga/main/ingest/pipeline.json

This file was deleted.

39 changes: 39 additions & 0 deletions filebeat/module/icinga/main/ingest/pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
description: Pipeline for parsing icinga main logs
processors:
- grok:
field: message
patterns:
- '\[%{TIMESTAMP:icinga.main.timestamp}\] %{WORD:log.level}/%{WORD:icinga.main.facility}:
%{GREEDYMULTILINE:message}'
ignore_missing: true
pattern_definitions:
TIMESTAMP: '%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}'
GREEDYMULTILINE: |-
(.|
)*
- date:
field: icinga.main.timestamp
target_field: '@timestamp'
formats:
- yyyy-MM-dd HH:mm:ss Z
ignore_failure: true
- remove:
field: icinga.main.timestamp
- set:
field: event.kind
value: event
- script:
lang: painless
source: >-
def errorLevels = ["warning", "critical"];
if (ctx?.log?.level != null) {
if (errorLevels.contains(ctx.log.level)) {
ctx.event.type = "error";
} else {
ctx.event.type = "info";
}
}
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
2 changes: 1 addition & 1 deletion filebeat/module/icinga/main/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ var:
os.windows:
- c:/programdata/icinga2/var/log/icinga2/icinga2.log*

ingest_pipeline: ingest/pipeline.json
ingest_pipeline: ingest/pipeline.yml
input: config/main.yml
6 changes: 6 additions & 0 deletions filebeat/module/icinga/main/test/test.log-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
{
"@timestamp": "2017-04-04T09:16:34.000Z",
"event.dataset": "icinga.main",
"event.kind": "event",
"event.module": "icinga",
"event.type": "info",
"fileset.name": "main",
"icinga.main.facility": "Notification",
"input.type": "log",
Expand All @@ -14,7 +16,9 @@
{
"@timestamp": "2017-04-04T09:16:34.000Z",
"event.dataset": "icinga.main",
"event.kind": "event",
"event.module": "icinga",
"event.type": "error",
"fileset.name": "main",
"icinga.main.facility": "PluginNotificationTask",
"input.type": "log",
Expand All @@ -29,7 +33,9 @@
{
"@timestamp": "2017-04-04T09:16:48.000Z",
"event.dataset": "icinga.main",
"event.kind": "event",
"event.module": "icinga",
"event.type": "info",
"fileset.name": "main",
"icinga.main.facility": "IdoMysqlConnection",
"input.type": "log",
Expand Down
21 changes: 0 additions & 21 deletions filebeat/module/icinga/startup/ingest/pipeline.json

This file was deleted.

29 changes: 29 additions & 0 deletions filebeat/module/icinga/startup/ingest/pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
description: Pipeline for parsing icinga startup logs
processors:
- grok:
field: message
patterns:
- '%{WORD:log.level}/%{WORD:icinga.startup.facility}: %{GREEDYMULTILINE:message}'
ignore_missing: true
pattern_definitions:
GREEDYMULTILINE: |-
(.|
)*
- set:
field: event.kind
value: event
- script:
lang: painless
source: >-
def errorLevels = ["warning", "critical"];
if (ctx?.log?.level != null) {
if (errorLevels.contains(ctx.log.level)) {
ctx.event.type = "error";
} else {
ctx.event.type = "info";
}
}
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
2 changes: 1 addition & 1 deletion filebeat/module/icinga/startup/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ var:
os.windows:
- c:/programdata/icinga2/var/log/icinga2/startup.log

ingest_pipeline: ingest/pipeline.json
ingest_pipeline: ingest/pipeline.yml
input: config/startup.yml
4 changes: 4 additions & 0 deletions filebeat/module/icinga/startup/test/test.log-expected.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
[
{
"event.dataset": "icinga.startup",
"event.kind": "event",
"event.module": "icinga",
"event.type": "info",
"fileset.name": "startup",
"icinga.startup.facility": "cli",
"input.type": "log",
Expand All @@ -12,7 +14,9 @@
},
{
"event.dataset": "icinga.startup",
"event.kind": "event",
"event.module": "icinga",
"event.type": "info",
"fileset.name": "startup",
"icinga.startup.facility": "cli",
"input.type": "log",
Expand Down

0 comments on commit 1051523

Please sign in to comment.