Skip to content

Commit

Permalink
Improve ECS categorization field mappings in redis module (#17918)
Browse files Browse the repository at this point in the history
- event.kind
- event.category
- event.type

Closes #16179
  • Loading branch information
leehinman authored Apr 27, 2020
1 parent ae1125d commit cc6c4e3
Show file tree
Hide file tree
Showing 9 changed files with 1,185 additions and 86 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Improve ECS categorization field mappings for nginx module. {issue}16174[16174] {pull}17844[17844]
- Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914]
- Improve ECS categorization field mappings in rabbitmq module. {issue}16178[16178] {pull}17916[17916]
- Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918]

*Heartbeat*

Expand Down
85 changes: 0 additions & 85 deletions filebeat/module/redis/log/ingest/pipeline.json

This file was deleted.

84 changes: 84 additions & 0 deletions filebeat/module/redis/log/ingest/pipeline.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
description: Pipeline for parsing redis logs
processors:
- grok:
field: message
patterns:
- (%{POSINT:process.pid:long}:%{CHAR:redis.log.role} )?(%{REDISTIMESTAMP1:redis.log.timestamp}||%{REDISTIMESTAMP2:redis.log.timestamp})
%{REDISLEVEL:log.level} %{GREEDYDATA:message}
- '%{POSINT:process.pid:long}:signal-handler \(%{POSINT:redis.log.timestamp}\)
%{GREEDYDATA:message}'
pattern_definitions:
CHAR: '[a-zA-Z]'
REDISLEVEL: '[.\-*#]'
REDISTIMESTAMP1: '%{MONTHDAY} %{MONTH} %{TIME}'
REDISTIMESTAMP2: '%{MONTHDAY} %{MONTH} %{YEAR} %{TIME}'
- script:
lang: painless
source: >-
if (ctx.log.level == params.dot) {
ctx.log.level = params.debug;
} else if (ctx.log.level == params.dash) {
ctx.log.level = params.verbose;
} else if (ctx.log.level == params.asterisk) {
ctx.log.level = params.notice;
} else if (ctx.log.level == params.hash) {
ctx.log.level = params.warning;
}
params:
dot: .
debug: debug
dash: '-'
verbose: verbose
asterisk: '*'
notice: notice
hash: '#'
warning: warning
- script:
lang: painless
source: >-
if (ctx.redis.log.role == params.master_abbrev) {
ctx.redis.log.role = params.master;
} else if (ctx.redis.log.role == params.slave_abbrev) {
ctx.redis.log.role = params.slave;
} else if (ctx.redis.log.role == params.child_abbrev) {
ctx.redis.log.role = params.child;
} else if (ctx.redis.log.role == params.sentinel_abbrev) {
ctx.redis.log.role = params.sentinel;
}
params:
master_abbrev: M
master: master
slave_abbrev: S
slave: slave
child_abbrev: C
child: child
sentinel_abbrev: X
sentinel: sentinel
- rename:
field: '@timestamp'
target_field: event.created
- date:
field: redis.log.timestamp
target_field: '@timestamp'
formats:
- dd MMM yyyy H:m:s.SSS
- dd MMM H:m:s.SSS
- dd MMM H:m:s
- UNIX
ignore_failure: true
- remove:
field: redis.log.timestamp
ignore_failure: true
- set:
field: event.kind
value: event
- append:
field: event.category
value: database
- append:
field: event.type
value: info
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
2 changes: 1 addition & 1 deletion filebeat/module/redis/log/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@ var:
os.windows:
- "c:/program files/Redis/logs/redis.log*"

ingest_pipeline: ingest/pipeline.json
ingest_pipeline: ingest/pipeline.yml
input: config/log.yml
7 changes: 7 additions & 0 deletions filebeat/module/redis/log/test/redis-5.0.3.log-expected.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
[
{
"event.category": [
"database"
],
"event.dataset": "redis.log",
"event.kind": "event",
"event.module": "redis",
"event.type": [
"info"
],
"fileset.name": "log",
"input.type": "log",
"log.level": "notice",
Expand Down
Loading

0 comments on commit cc6c4e3

Please sign in to comment.