From d21b351cdacacd4bcf520c0a918e98eda18d0587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 30 May 2018 11:04:25 +0200 Subject: [PATCH] Comply with PostgreSQL database name format (#7198) Support `_` and `$` in PostgreSQL database names as all of those are valid characters in a name. Reference: > SQL identifiers and key words must begin with a letter (a-z, but also letters with diacritical marks and non-Latin letters) or an underscore (_). Subsequent characters in an identifier or key word can be letters, underscores, digits (0-9), or dollar signs ($). Note that dollar signs are not allowed in identifiers according to the letter of the SQL standard, so their use might render applications less portable. (Source: https://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS) Closes #7195 --- CHANGELOG.asciidoc | 1 + filebeat/module/postgresql/log/ingest/pipeline.json | 11 ++++++----- .../log/test/postgresql-9.6-debian-with-slowlog.log | 6 +++--- ...stgresql-9.6-debian-with-slowlog.log-expected.json | 6 +++--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6b176e5d49e2..bf8ef4dd81f7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -81,6 +81,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Commit registry writes to stable storage to avoid corrupt registry files. {issue}6792[6792] - Fix a data race between stopping and starting of the harverters. {issue}#6879[6879] - Fix a parsing issue in the syslog input for RFC3339 timestamp and time with nanoseconds. {pull}7046[7046] +- Comply with PostgreSQL database name format {pull}7198[7198] *Heartbeat* - Fix race due to updates of shared a map, that was not supposed to be shared between multiple go-routines. {issue}6616[6616] diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 17e461d897e5..1d1904b1e384 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,16 +6,17 @@ "field": "message", "ignore_missing": true, "patterns": [ - "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{HOSTNAME:postgresql.log.database} %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}", - "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{HOSTNAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}", - "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{HOSTNAME:postgresql.log.database} %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}", - "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{HOSTNAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}", + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database} %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}", + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: duration: %{NUMBER:postgresql.log.duration} ms statement: %{MULTILINEQUERY:postgresql.log.query}", + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database} %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}", + "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] \\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\] %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}", "%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] %{WORD:postgresql.log.level}: ?%{GREEDYDATA:postgresql.log.message}" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", "GREEDYDATA": ".*", - "MULTILINEQUERY" : "(.|\n|\t)*?;$" + "MULTILINEQUERY" : "(.|\n|\t)*?;$", + "POSTGRESQL_DB_NAME": "[a-zA-Z0-9_]+[a-zA-Z0-9_\\$]*" } } }, diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log index 1903a2a16db8..39a4d0ffa251 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log @@ -56,6 +56,6 @@ AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 1,2; 2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null); -2017-07-31 13:46:02.670 CEST [5502] postgres@clients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now()); -2017-07-31 13:46:23.016 CEST [5502] postgres@clients LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now()); -2017-07-31 13:46:55.637 CEST [5502] postgres@clients LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null); +2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now()); +2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now()); +2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null); diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index 4e49ce2b38a3..9c001e737c76 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -732,7 +732,7 @@ "log": { "duration": "10.540", "thread_id": "5502", - "database": "clients", + "database": "c$lients", "level": "LOG", "timezone": "CEST", "query": "insert into cats(name, toy, born) values('kate', 'ball', now());", @@ -781,7 +781,7 @@ "log": { "duration": "5.156", "thread_id": "5502", - "database": "clients", + "database": "_clients$db", "level": "LOG", "timezone": "CEST", "query": "insert into cats(name, toy, born) values('frida', 'horse', now());", @@ -830,7 +830,7 @@ "log": { "duration": "25.871", "thread_id": "5502", - "database": "clients", + "database": "clients_db", "level": "LOG", "timezone": "CEST", "query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);",