From 6b84f56d207f0e3e5ab2990a9ba82029905df6ff Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 29 Nov 2018 15:50:59 -0500 Subject: [PATCH 01/11] Perform straightforward ECS renames: - postgresql.log.level => log.level - postgresql.log.user => user.name --- .../postgresql/log/ingest/pipeline.json | 2 +- ...-9.6-debian-with-slowlog.log-expected.json | 85 +++++++++---------- ...ostgresql-9.6-multi-core.log-expected.json | 47 +++++----- 3 files changed, 60 insertions(+), 74 deletions(-) diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 398b1d95405..5b4a74a3c44 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,7 +6,7 @@ "field": "message", "ignore_missing": true, "patterns": [ - "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:postgresql.log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:postgresql.log.message})" + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index 4598f1e87b9..106e1022e57 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -4,10 +4,9 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 0, - "message": "2017-07-31 13:36:42.585 CEST [4974] LOG: database system was shut down at 2017-06-17 16:58:04 CEST", - "postgresql.log.level": "LOG", - "postgresql.log.message": "database system was shut down at 2017-06-17 16:58:04 CEST", + "message": "database system was shut down at 2017-06-17 16:58:04 CEST", "postgresql.log.thread_id": "4974", "postgresql.log.timestamp": "2017-07-31 13:36:42.585", "postgresql.log.timezone": "CEST" @@ -17,10 +16,9 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 100, - "message": "2017-07-31 13:36:42.605 CEST [4974] LOG: MultiXact member wraparound protections are now enabled", - "postgresql.log.level": "LOG", - "postgresql.log.message": "MultiXact member wraparound protections are now enabled", + "message": "MultiXact member wraparound protections are now enabled", "postgresql.log.thread_id": "4974", "postgresql.log.timestamp": "2017-07-31 13:36:42.605", "postgresql.log.timezone": "CEST" @@ -30,10 +28,9 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 198, - "message": "2017-07-31 13:36:42.615 CEST [4978] LOG: autovacuum launcher started", - "postgresql.log.level": "LOG", - "postgresql.log.message": "autovacuum launcher started", + "message": "autovacuum launcher started", "postgresql.log.thread_id": "4978", "postgresql.log.timestamp": "2017-07-31 13:36:42.615", "postgresql.log.timezone": "CEST" @@ -43,10 +40,9 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 268, - "message": "2017-07-31 13:36:42.616 CEST [4973] LOG: database system is ready to accept connections", - "postgresql.log.level": "LOG", - "postgresql.log.message": "database system is ready to accept connections", + "message": "database system is ready to accept connections", "postgresql.log.thread_id": "4973", "postgresql.log.timestamp": "2017-07-31 13:36:42.616", "postgresql.log.timezone": "CEST" @@ -56,15 +52,14 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 357, - "message": "2017-07-31 13:36:42.956 CEST [4980] [unknown]@[unknown] LOG: incomplete startup packet", + "message": "incomplete startup packet", "postgresql.log.database": "unknown", - "postgresql.log.level": "LOG", - "postgresql.log.message": "incomplete startup packet", "postgresql.log.thread_id": "4980", "postgresql.log.timestamp": "2017-07-31 13:36:42.956", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "unknown" + "user.name": "unknown" }, { "@timestamp": "2017-07-31T13:36:43.557Z", @@ -74,16 +69,16 @@ "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 445, "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", "postgresql.log.duration": "37.118", - "postgresql.log.level": "LOG", "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.thread_id": "4983", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:36:44.104Z", @@ -93,16 +88,16 @@ "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 873, "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", "postgresql.log.duration": "2.895", - "postgresql.log.level": "LOG", "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.thread_id": "4986", "postgresql.log.timestamp": "2017-07-31 13:36:44.104", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:36:44.642Z", @@ -112,46 +107,44 @@ "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 1300, "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", "postgresql.log.duration": "2.809", - "postgresql.log.level": "LOG", "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.thread_id": "4989", "postgresql.log.timestamp": "2017-07-31 13:36:44.642", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:16.249Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "FATAL", "log.offset": 1727, - "message": "2017-07-31 13:39:16.249 CEST [5407] postgres@users FATAL: database \"users\" does not exist", + "message": "database \"users\" does not exist", "postgresql.log.database": "users", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "database \"users\" does not exist", "postgresql.log.thread_id": "5407", "postgresql.log.timestamp": "2017-07-31 13:39:16.249", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:17.945Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "FATAL", "log.offset": 1818, - "message": "2017-07-31 13:39:17.945 CEST [5500] postgres@user FATAL: database \"user\" does not exist", + "message": "database \"user\" does not exist", "postgresql.log.database": "user", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "database \"user\" does not exist", "postgresql.log.thread_id": "5500", "postgresql.log.timestamp": "2017-07-31 13:39:17.945", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:21.025Z", @@ -161,48 +154,48 @@ "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 1907, "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.database": "postgres", "postgresql.log.duration": "37.598", - "postgresql.log.level": "LOG", "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-07-31 13:39:21.025", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:31.619Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 2620, "message": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", "postgresql.log.database": "clients", "postgresql.log.duration": "9.482", - "postgresql.log.level": "LOG", "postgresql.log.query": "select * from clients;", "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:39:31.619", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:40.147Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 2733, "message": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;", "postgresql.log.database": "clients", "postgresql.log.duration": "0.765", - "postgresql.log.level": "LOG", "postgresql.log.query": "select id from clients;", "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:39:40.147", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:40:54.310Z", @@ -212,79 +205,79 @@ "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 2847, "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.database": "clients", "postgresql.log.duration": "26.082", - "postgresql.log.level": "LOG", "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:40:54.310", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:43:22.645Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 3559, "message": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.database": "clients", "postgresql.log.duration": "36.162", - "postgresql.log.level": "LOG", "postgresql.log.query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:43:22.645", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:02.670Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 3751, "message": "2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.database": "c$lients", "postgresql.log.duration": "10.540", - "postgresql.log.level": "LOG", "postgresql.log.query": "insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:23.016Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 3908, "message": "2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.database": "_clients$db", "postgresql.log.duration": "5.156", - "postgresql.log.level": "LOG", "postgresql.log.query": "insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:55.637Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 4069, "message": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.database": "clients_db", "postgresql.log.duration": "25.871", - "postgresql.log.level": "LOG", "postgresql.log.query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" } ] \ No newline at end of file diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json index 223846890a4..fd63499fcda 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json @@ -4,32 +4,30 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 0, - "message": "2017-04-03 22:32:14.322 CEST [12975-1] [unknown]@[unknown] LOG: incomplete startup packet", + "message": "incomplete startup packet", "postgresql.log.core_id": "1", "postgresql.log.database": "unknown", - "postgresql.log.level": "LOG", - "postgresql.log.message": "incomplete startup packet", "postgresql.log.thread_id": "12975", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "unknown" + "user.name": "unknown" }, { "@timestamp": "2017-04-03T22:32:14.322Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "FATAL", "log.offset": 91, - "message": "2017-04-03 22:32:14.322 CEST [5404-1] postgres@user FATAL: database \"user\" does not exist", + "message": "database \"user\" does not exist", "postgresql.log.core_id": "1", "postgresql.log.database": "user", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "database \"user\" does not exist", "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-04-03T22:35:22.389Z", @@ -39,28 +37,27 @@ "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 182, "message": "2017-04-03 22:35:22.389 CEST [5404-2] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.core_id": "2", "postgresql.log.database": "postgres", "postgresql.log.duration": "37.598", - "postgresql.log.level": "LOG", "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-04-03 22:35:22.389", "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:36:43.557Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 897, - "message": "2017-07-31 13:36:43.557 EST [835-1] LOG: autovacuum launcher started", + "message": "autovacuum launcher started", "postgresql.log.core_id": "1", - "postgresql.log.level": "LOG", - "postgresql.log.message": "autovacuum launcher started", "postgresql.log.thread_id": "835", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", "postgresql.log.timezone": "EST" @@ -70,11 +67,10 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "LOG", "log.offset": 967, - "message": "2017-07-31 13:36:44.227 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart)", + "message": "checkpoints are occurring too frequently (25 seconds apart)", "postgresql.log.core_id": "1", - "postgresql.log.level": "LOG", - "postgresql.log.message": "checkpoints are occurring too frequently (25 seconds apart)", "postgresql.log.thread_id": "832", "postgresql.log.timestamp": "2017-07-31 13:36:44.227", "postgresql.log.timezone": "EST" @@ -84,11 +80,10 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "HINT", "log.offset": 1069, - "message": "2017-07-31 13:46:02.670 EST [832-2] HINT: Consider increasing the configuration parameter \"max_wal_size\".", + "message": "Consider increasing the configuration parameter \"max_wal_size\".", "postgresql.log.core_id": "2", - "postgresql.log.level": "HINT", - "postgresql.log.message": "Consider increasing the configuration parameter \"max_wal_size\".", "postgresql.log.thread_id": "832", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", "postgresql.log.timezone": "EST" @@ -98,31 +93,29 @@ "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "FATAL", "log.offset": 1176, - "message": "2017-07-31 13:46:23.016 EST [768-1] postgres@postgres FATAL: the database system is starting up", + "message": "the database system is starting up", "postgresql.log.core_id": "1", "postgresql.log.database": "postgres", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "the database system is starting up", "postgresql.log.thread_id": "768", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "postgresql.log.timezone": "EST", - "postgresql.log.user": "postgres" + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:55.637Z", "event.dataset": "log", "event.module": "postgresql", "input.type": "log", + "log.level": "FATAL", "log.offset": 1273, - "message": "2017-07-31 13:46:55.637 EST [771-1] postgres@postgres FATAL: the database system is starting up", + "message": "the database system is starting up", "postgresql.log.core_id": "1", "postgresql.log.database": "postgres", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "the database system is starting up", "postgresql.log.thread_id": "771", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "postgresql.log.timezone": "EST", - "postgresql.log.user": "postgres" + "user.name": "postgres" } ] \ No newline at end of file From 39053ff759d4263e95f835b6f58883d4d4406f86 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 30 Nov 2018 08:58:34 -0500 Subject: [PATCH 02/11] Rename incorrectly named `postgresql.log.thread_id` to `process.pid` --- .../postgresql/log/ingest/pipeline.json | 2 +- ...-9.6-debian-with-slowlog.log-expected.json | 44 +++++++++---------- ...ostgresql-9.6-multi-core.log-expected.json | 22 +++++----- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 5b4a74a3c44..d2db3455f2e 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,7 +6,7 @@ "field": "message", "ignore_missing": true, "patterns": [ - "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:process.pid}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index 106e1022e57..ce33876669e 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -7,9 +7,9 @@ "log.level": "LOG", "log.offset": 0, "message": "database system was shut down at 2017-06-17 16:58:04 CEST", - "postgresql.log.thread_id": "4974", "postgresql.log.timestamp": "2017-07-31 13:36:42.585", - "postgresql.log.timezone": "CEST" + "postgresql.log.timezone": "CEST", + "process.pid": "4974" }, { "@timestamp": "2017-07-31T13:36:42.605Z", @@ -19,9 +19,9 @@ "log.level": "LOG", "log.offset": 100, "message": "MultiXact member wraparound protections are now enabled", - "postgresql.log.thread_id": "4974", "postgresql.log.timestamp": "2017-07-31 13:36:42.605", - "postgresql.log.timezone": "CEST" + "postgresql.log.timezone": "CEST", + "process.pid": "4974" }, { "@timestamp": "2017-07-31T13:36:42.615Z", @@ -31,9 +31,9 @@ "log.level": "LOG", "log.offset": 198, "message": "autovacuum launcher started", - "postgresql.log.thread_id": "4978", "postgresql.log.timestamp": "2017-07-31 13:36:42.615", - "postgresql.log.timezone": "CEST" + "postgresql.log.timezone": "CEST", + "process.pid": "4978" }, { "@timestamp": "2017-07-31T13:36:42.616Z", @@ -43,9 +43,9 @@ "log.level": "LOG", "log.offset": 268, "message": "database system is ready to accept connections", - "postgresql.log.thread_id": "4973", "postgresql.log.timestamp": "2017-07-31 13:36:42.616", - "postgresql.log.timezone": "CEST" + "postgresql.log.timezone": "CEST", + "process.pid": "4973" }, { "@timestamp": "2017-07-31T13:36:42.956Z", @@ -56,9 +56,9 @@ "log.offset": 357, "message": "incomplete startup packet", "postgresql.log.database": "unknown", - "postgresql.log.thread_id": "4980", "postgresql.log.timestamp": "2017-07-31 13:36:42.956", "postgresql.log.timezone": "CEST", + "process.pid": "4980", "user.name": "unknown" }, { @@ -75,9 +75,9 @@ "postgresql.log.database": "postgres", "postgresql.log.duration": "37.118", "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", - "postgresql.log.thread_id": "4983", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", "postgresql.log.timezone": "CEST", + "process.pid": "4983", "user.name": "postgres" }, { @@ -94,9 +94,9 @@ "postgresql.log.database": "postgres", "postgresql.log.duration": "2.895", "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", - "postgresql.log.thread_id": "4986", "postgresql.log.timestamp": "2017-07-31 13:36:44.104", "postgresql.log.timezone": "CEST", + "process.pid": "4986", "user.name": "postgres" }, { @@ -113,9 +113,9 @@ "postgresql.log.database": "postgres", "postgresql.log.duration": "2.809", "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", - "postgresql.log.thread_id": "4989", "postgresql.log.timestamp": "2017-07-31 13:36:44.642", "postgresql.log.timezone": "CEST", + "process.pid": "4989", "user.name": "postgres" }, { @@ -127,9 +127,9 @@ "log.offset": 1727, "message": "database \"users\" does not exist", "postgresql.log.database": "users", - "postgresql.log.thread_id": "5407", "postgresql.log.timestamp": "2017-07-31 13:39:16.249", "postgresql.log.timezone": "CEST", + "process.pid": "5407", "user.name": "postgres" }, { @@ -141,9 +141,9 @@ "log.offset": 1818, "message": "database \"user\" does not exist", "postgresql.log.database": "user", - "postgresql.log.thread_id": "5500", "postgresql.log.timestamp": "2017-07-31 13:39:17.945", "postgresql.log.timezone": "CEST", + "process.pid": "5500", "user.name": "postgres" }, { @@ -160,9 +160,9 @@ "postgresql.log.database": "postgres", "postgresql.log.duration": "37.598", "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", - "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-07-31 13:39:21.025", "postgresql.log.timezone": "CEST", + "process.pid": "5404", "user.name": "postgres" }, { @@ -176,9 +176,9 @@ "postgresql.log.database": "clients", "postgresql.log.duration": "9.482", "postgresql.log.query": "select * from clients;", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:39:31.619", "postgresql.log.timezone": "CEST", + "process.pid": "5502", "user.name": "postgres" }, { @@ -192,9 +192,9 @@ "postgresql.log.database": "clients", "postgresql.log.duration": "0.765", "postgresql.log.query": "select id from clients;", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:39:40.147", "postgresql.log.timezone": "CEST", + "process.pid": "5502", "user.name": "postgres" }, { @@ -211,9 +211,9 @@ "postgresql.log.database": "clients", "postgresql.log.duration": "26.082", "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:40:54.310", "postgresql.log.timezone": "CEST", + "process.pid": "5502", "user.name": "postgres" }, { @@ -227,9 +227,9 @@ "postgresql.log.database": "clients", "postgresql.log.duration": "36.162", "postgresql.log.query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:43:22.645", "postgresql.log.timezone": "CEST", + "process.pid": "5502", "user.name": "postgres" }, { @@ -243,9 +243,9 @@ "postgresql.log.database": "c$lients", "postgresql.log.duration": "10.540", "postgresql.log.query": "insert into cats(name, toy, born) values('kate', 'ball', now());", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", "postgresql.log.timezone": "CEST", + "process.pid": "5502", "user.name": "postgres" }, { @@ -259,9 +259,9 @@ "postgresql.log.database": "_clients$db", "postgresql.log.duration": "5.156", "postgresql.log.query": "insert into cats(name, toy, born) values('frida', 'horse', now());", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "postgresql.log.timezone": "CEST", + "process.pid": "5502", "user.name": "postgres" }, { @@ -275,9 +275,9 @@ "postgresql.log.database": "clients_db", "postgresql.log.duration": "25.871", "postgresql.log.query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "postgresql.log.timezone": "CEST", + "process.pid": "5502", "user.name": "postgres" } ] \ No newline at end of file diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json index fd63499fcda..60d39578b2d 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json @@ -9,9 +9,9 @@ "message": "incomplete startup packet", "postgresql.log.core_id": "1", "postgresql.log.database": "unknown", - "postgresql.log.thread_id": "12975", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", "postgresql.log.timezone": "CEST", + "process.pid": "12975", "user.name": "unknown" }, { @@ -24,9 +24,9 @@ "message": "database \"user\" does not exist", "postgresql.log.core_id": "1", "postgresql.log.database": "user", - "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", "postgresql.log.timezone": "CEST", + "process.pid": "5404", "user.name": "postgres" }, { @@ -44,9 +44,9 @@ "postgresql.log.database": "postgres", "postgresql.log.duration": "37.598", "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", - "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-04-03 22:35:22.389", "postgresql.log.timezone": "CEST", + "process.pid": "5404", "user.name": "postgres" }, { @@ -58,9 +58,9 @@ "log.offset": 897, "message": "autovacuum launcher started", "postgresql.log.core_id": "1", - "postgresql.log.thread_id": "835", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", - "postgresql.log.timezone": "EST" + "postgresql.log.timezone": "EST", + "process.pid": "835" }, { "@timestamp": "2017-07-31T13:36:44.227Z", @@ -71,9 +71,9 @@ "log.offset": 967, "message": "checkpoints are occurring too frequently (25 seconds apart)", "postgresql.log.core_id": "1", - "postgresql.log.thread_id": "832", "postgresql.log.timestamp": "2017-07-31 13:36:44.227", - "postgresql.log.timezone": "EST" + "postgresql.log.timezone": "EST", + "process.pid": "832" }, { "@timestamp": "2017-07-31T13:46:02.670Z", @@ -84,9 +84,9 @@ "log.offset": 1069, "message": "Consider increasing the configuration parameter \"max_wal_size\".", "postgresql.log.core_id": "2", - "postgresql.log.thread_id": "832", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", - "postgresql.log.timezone": "EST" + "postgresql.log.timezone": "EST", + "process.pid": "832" }, { "@timestamp": "2017-07-31T13:46:23.016Z", @@ -98,9 +98,9 @@ "message": "the database system is starting up", "postgresql.log.core_id": "1", "postgresql.log.database": "postgres", - "postgresql.log.thread_id": "768", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "postgresql.log.timezone": "EST", + "process.pid": "768", "user.name": "postgres" }, { @@ -113,9 +113,9 @@ "message": "the database system is starting up", "postgresql.log.core_id": "1", "postgresql.log.database": "postgres", - "postgresql.log.thread_id": "771", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "postgresql.log.timezone": "EST", + "process.pid": "771", "user.name": "postgres" } ] \ No newline at end of file From d41e59922dbceb50c3ae937d0a2dea377435d18b Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 30 Nov 2018 09:03:55 -0500 Subject: [PATCH 03/11] Coercions: - PID to int - duration to float --- .../postgresql/log/ingest/pipeline.json | 2 +- ...-9.6-debian-with-slowlog.log-expected.json | 36 +++++++++---------- ...ostgresql-9.6-multi-core.log-expected.json | 16 ++++----- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index d2db3455f2e..116e8355ae6 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,7 +6,7 @@ "field": "message", "ignore_missing": true, "patterns": [ - "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:process.pid}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index ce33876669e..442116c0492 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -9,7 +9,7 @@ "message": "database system was shut down at 2017-06-17 16:58:04 CEST", "postgresql.log.timestamp": "2017-07-31 13:36:42.585", "postgresql.log.timezone": "CEST", - "process.pid": "4974" + "process.pid": 4974 }, { "@timestamp": "2017-07-31T13:36:42.605Z", @@ -21,7 +21,7 @@ "message": "MultiXact member wraparound protections are now enabled", "postgresql.log.timestamp": "2017-07-31 13:36:42.605", "postgresql.log.timezone": "CEST", - "process.pid": "4974" + "process.pid": 4974 }, { "@timestamp": "2017-07-31T13:36:42.615Z", @@ -33,7 +33,7 @@ "message": "autovacuum launcher started", "postgresql.log.timestamp": "2017-07-31 13:36:42.615", "postgresql.log.timezone": "CEST", - "process.pid": "4978" + "process.pid": 4978 }, { "@timestamp": "2017-07-31T13:36:42.616Z", @@ -45,7 +45,7 @@ "message": "database system is ready to accept connections", "postgresql.log.timestamp": "2017-07-31 13:36:42.616", "postgresql.log.timezone": "CEST", - "process.pid": "4973" + "process.pid": 4973 }, { "@timestamp": "2017-07-31T13:36:42.956Z", @@ -58,7 +58,7 @@ "postgresql.log.database": "unknown", "postgresql.log.timestamp": "2017-07-31 13:36:42.956", "postgresql.log.timezone": "CEST", - "process.pid": "4980", + "process.pid": 4980, "user.name": "unknown" }, { @@ -77,7 +77,7 @@ "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", "postgresql.log.timezone": "CEST", - "process.pid": "4983", + "process.pid": 4983, "user.name": "postgres" }, { @@ -96,7 +96,7 @@ "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:44.104", "postgresql.log.timezone": "CEST", - "process.pid": "4986", + "process.pid": 4986, "user.name": "postgres" }, { @@ -115,7 +115,7 @@ "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:44.642", "postgresql.log.timezone": "CEST", - "process.pid": "4989", + "process.pid": 4989, "user.name": "postgres" }, { @@ -129,7 +129,7 @@ "postgresql.log.database": "users", "postgresql.log.timestamp": "2017-07-31 13:39:16.249", "postgresql.log.timezone": "CEST", - "process.pid": "5407", + "process.pid": 5407, "user.name": "postgres" }, { @@ -143,7 +143,7 @@ "postgresql.log.database": "user", "postgresql.log.timestamp": "2017-07-31 13:39:17.945", "postgresql.log.timezone": "CEST", - "process.pid": "5500", + "process.pid": 5500, "user.name": "postgres" }, { @@ -162,7 +162,7 @@ "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-07-31 13:39:21.025", "postgresql.log.timezone": "CEST", - "process.pid": "5404", + "process.pid": 5404, "user.name": "postgres" }, { @@ -178,7 +178,7 @@ "postgresql.log.query": "select * from clients;", "postgresql.log.timestamp": "2017-07-31 13:39:31.619", "postgresql.log.timezone": "CEST", - "process.pid": "5502", + "process.pid": 5502, "user.name": "postgres" }, { @@ -194,7 +194,7 @@ "postgresql.log.query": "select id from clients;", "postgresql.log.timestamp": "2017-07-31 13:39:40.147", "postgresql.log.timezone": "CEST", - "process.pid": "5502", + "process.pid": 5502, "user.name": "postgres" }, { @@ -213,7 +213,7 @@ "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-07-31 13:40:54.310", "postgresql.log.timezone": "CEST", - "process.pid": "5502", + "process.pid": 5502, "user.name": "postgres" }, { @@ -229,7 +229,7 @@ "postgresql.log.query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.timestamp": "2017-07-31 13:43:22.645", "postgresql.log.timezone": "CEST", - "process.pid": "5502", + "process.pid": 5502, "user.name": "postgres" }, { @@ -245,7 +245,7 @@ "postgresql.log.query": "insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", "postgresql.log.timezone": "CEST", - "process.pid": "5502", + "process.pid": 5502, "user.name": "postgres" }, { @@ -261,7 +261,7 @@ "postgresql.log.query": "insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "postgresql.log.timezone": "CEST", - "process.pid": "5502", + "process.pid": 5502, "user.name": "postgres" }, { @@ -277,7 +277,7 @@ "postgresql.log.query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "postgresql.log.timezone": "CEST", - "process.pid": "5502", + "process.pid": 5502, "user.name": "postgres" } ] \ No newline at end of file diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json index 60d39578b2d..919723ee2b9 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json @@ -11,7 +11,7 @@ "postgresql.log.database": "unknown", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", "postgresql.log.timezone": "CEST", - "process.pid": "12975", + "process.pid": 12975, "user.name": "unknown" }, { @@ -26,7 +26,7 @@ "postgresql.log.database": "user", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", "postgresql.log.timezone": "CEST", - "process.pid": "5404", + "process.pid": 5404, "user.name": "postgres" }, { @@ -46,7 +46,7 @@ "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-04-03 22:35:22.389", "postgresql.log.timezone": "CEST", - "process.pid": "5404", + "process.pid": 5404, "user.name": "postgres" }, { @@ -60,7 +60,7 @@ "postgresql.log.core_id": "1", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", "postgresql.log.timezone": "EST", - "process.pid": "835" + "process.pid": 835 }, { "@timestamp": "2017-07-31T13:36:44.227Z", @@ -73,7 +73,7 @@ "postgresql.log.core_id": "1", "postgresql.log.timestamp": "2017-07-31 13:36:44.227", "postgresql.log.timezone": "EST", - "process.pid": "832" + "process.pid": 832 }, { "@timestamp": "2017-07-31T13:46:02.670Z", @@ -86,7 +86,7 @@ "postgresql.log.core_id": "2", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", "postgresql.log.timezone": "EST", - "process.pid": "832" + "process.pid": 832 }, { "@timestamp": "2017-07-31T13:46:23.016Z", @@ -100,7 +100,7 @@ "postgresql.log.database": "postgres", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "postgresql.log.timezone": "EST", - "process.pid": "768", + "process.pid": 768, "user.name": "postgres" }, { @@ -115,7 +115,7 @@ "postgresql.log.database": "postgres", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "postgresql.log.timezone": "EST", - "process.pid": "771", + "process.pid": 771, "user.name": "postgres" } ] \ No newline at end of file From 7110322dea34f2ec58414b61fb00d356b010040f Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 30 Nov 2018 09:31:23 -0500 Subject: [PATCH 04/11] Copy duration to `event.duration`, properly scaled --- filebeat/module/postgresql/log/ingest/pipeline.json | 8 ++++++++ ...stgresql-9.6-debian-with-slowlog.log-expected.json | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 116e8355ae6..e5525f15c4e 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -24,6 +24,14 @@ ], "ignore_failure": true } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.postgresql.log.duration * params.scale)", + "params": { "scale": 1000000 }, + "if": "ctx.postgresql.log.containsKey('duration')" + } } ], "on_failure": [ diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index 442116c0492..b9b1fc458f4 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -64,6 +64,7 @@ { "@timestamp": "2017-07-31T13:36:43.557Z", "event.dataset": "log", + "event.duration": 37118000, "event.module": "postgresql", "input.type": "log", "log.flags": [ @@ -83,6 +84,7 @@ { "@timestamp": "2017-07-31T13:36:44.104Z", "event.dataset": "log", + "event.duration": 2895000, "event.module": "postgresql", "input.type": "log", "log.flags": [ @@ -102,6 +104,7 @@ { "@timestamp": "2017-07-31T13:36:44.642Z", "event.dataset": "log", + "event.duration": 2809000, "event.module": "postgresql", "input.type": "log", "log.flags": [ @@ -149,6 +152,7 @@ { "@timestamp": "2017-07-31T13:39:21.025Z", "event.dataset": "log", + "event.duration": 37598000, "event.module": "postgresql", "input.type": "log", "log.flags": [ @@ -168,6 +172,7 @@ { "@timestamp": "2017-07-31T13:39:31.619Z", "event.dataset": "log", + "event.duration": 9482000, "event.module": "postgresql", "input.type": "log", "log.level": "LOG", @@ -184,6 +189,7 @@ { "@timestamp": "2017-07-31T13:39:40.147Z", "event.dataset": "log", + "event.duration": 765000, "event.module": "postgresql", "input.type": "log", "log.level": "LOG", @@ -200,6 +206,7 @@ { "@timestamp": "2017-07-31T13:40:54.310Z", "event.dataset": "log", + "event.duration": 26082001, "event.module": "postgresql", "input.type": "log", "log.flags": [ @@ -219,6 +226,7 @@ { "@timestamp": "2017-07-31T13:43:22.645Z", "event.dataset": "log", + "event.duration": 36161999, "event.module": "postgresql", "input.type": "log", "log.level": "LOG", @@ -235,6 +243,7 @@ { "@timestamp": "2017-07-31T13:46:02.670Z", "event.dataset": "log", + "event.duration": 10540000, "event.module": "postgresql", "input.type": "log", "log.level": "LOG", @@ -251,6 +260,7 @@ { "@timestamp": "2017-07-31T13:46:23.016Z", "event.dataset": "log", + "event.duration": 5156000, "event.module": "postgresql", "input.type": "log", "log.level": "LOG", @@ -267,6 +277,7 @@ { "@timestamp": "2017-07-31T13:46:55.637Z", "event.dataset": "log", + "event.duration": 25871000, "event.module": "postgresql", "input.type": "log", "log.level": "LOG", From e2cf01e598afd0e290851812b0ee17f7d847e7b7 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 14:17:13 -0500 Subject: [PATCH 05/11] Add float coercion missed during rebase --- .../postgresql/log/ingest/pipeline.json | 2 +- ...-9.6-debian-with-slowlog.log-expected.json | 22 +++++++++---------- ...ostgresql-9.6-multi-core.log-expected.json | 3 ++- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index e5525f15c4e..4782f92521e 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,7 +6,7 @@ "field": "message", "ignore_missing": true, "patterns": [ - "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index b9b1fc458f4..5c3107f8929 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -74,7 +74,7 @@ "log.offset": 445, "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": "37.118", + "postgresql.log.duration": 37.118, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", "postgresql.log.timezone": "CEST", @@ -94,7 +94,7 @@ "log.offset": 873, "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": "2.895", + "postgresql.log.duration": 2.895, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:44.104", "postgresql.log.timezone": "CEST", @@ -114,7 +114,7 @@ "log.offset": 1300, "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": "2.809", + "postgresql.log.duration": 2.809, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:44.642", "postgresql.log.timezone": "CEST", @@ -162,7 +162,7 @@ "log.offset": 1907, "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.database": "postgres", - "postgresql.log.duration": "37.598", + "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-07-31 13:39:21.025", "postgresql.log.timezone": "CEST", @@ -179,7 +179,7 @@ "log.offset": 2620, "message": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", "postgresql.log.database": "clients", - "postgresql.log.duration": "9.482", + "postgresql.log.duration": 9.482, "postgresql.log.query": "select * from clients;", "postgresql.log.timestamp": "2017-07-31 13:39:31.619", "postgresql.log.timezone": "CEST", @@ -196,7 +196,7 @@ "log.offset": 2733, "message": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;", "postgresql.log.database": "clients", - "postgresql.log.duration": "0.765", + "postgresql.log.duration": 0.765, "postgresql.log.query": "select id from clients;", "postgresql.log.timestamp": "2017-07-31 13:39:40.147", "postgresql.log.timezone": "CEST", @@ -216,7 +216,7 @@ "log.offset": 2847, "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.database": "clients", - "postgresql.log.duration": "26.082", + "postgresql.log.duration": 26.082, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-07-31 13:40:54.310", "postgresql.log.timezone": "CEST", @@ -233,7 +233,7 @@ "log.offset": 3559, "message": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.database": "clients", - "postgresql.log.duration": "36.162", + "postgresql.log.duration": 36.162, "postgresql.log.query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.timestamp": "2017-07-31 13:43:22.645", "postgresql.log.timezone": "CEST", @@ -250,7 +250,7 @@ "log.offset": 3751, "message": "2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.database": "c$lients", - "postgresql.log.duration": "10.540", + "postgresql.log.duration": 10.54, "postgresql.log.query": "insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", "postgresql.log.timezone": "CEST", @@ -267,7 +267,7 @@ "log.offset": 3908, "message": "2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.database": "_clients$db", - "postgresql.log.duration": "5.156", + "postgresql.log.duration": 5.156, "postgresql.log.query": "insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "postgresql.log.timezone": "CEST", @@ -284,7 +284,7 @@ "log.offset": 4069, "message": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.database": "clients_db", - "postgresql.log.duration": "25.871", + "postgresql.log.duration": 25.871, "postgresql.log.query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "postgresql.log.timezone": "CEST", diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json index 919723ee2b9..c7b7537c046 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json @@ -32,6 +32,7 @@ { "@timestamp": "2017-04-03T22:35:22.389Z", "event.dataset": "log", + "event.duration": 37598000, "event.module": "postgresql", "input.type": "log", "log.flags": [ @@ -42,7 +43,7 @@ "message": "2017-04-03 22:35:22.389 CEST [5404-2] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.core_id": "2", "postgresql.log.database": "postgres", - "postgresql.log.duration": "37.598", + "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-04-03 22:35:22.389", "postgresql.log.timezone": "CEST", From b35ce71f456297e09243d7a5045ab67f16a18fb1 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 14:25:16 -0500 Subject: [PATCH 06/11] Coerce `core_id` to an integer as well --- .../module/postgresql/log/ingest/pipeline.json | 2 +- .../postgresql-9.6-multi-core.log-expected.json | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 4782f92521e..9427ff91a63 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,7 +6,7 @@ "field": "message", "ignore_missing": true, "patterns": [ - "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json index c7b7537c046..9c419cb3242 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json @@ -7,7 +7,7 @@ "log.level": "LOG", "log.offset": 0, "message": "incomplete startup packet", - "postgresql.log.core_id": "1", + "postgresql.log.core_id": 1, "postgresql.log.database": "unknown", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", "postgresql.log.timezone": "CEST", @@ -22,7 +22,7 @@ "log.level": "FATAL", "log.offset": 91, "message": "database \"user\" does not exist", - "postgresql.log.core_id": "1", + "postgresql.log.core_id": 1, "postgresql.log.database": "user", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", "postgresql.log.timezone": "CEST", @@ -41,7 +41,7 @@ "log.level": "LOG", "log.offset": 182, "message": "2017-04-03 22:35:22.389 CEST [5404-2] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", - "postgresql.log.core_id": "2", + "postgresql.log.core_id": 2, "postgresql.log.database": "postgres", "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", @@ -58,7 +58,7 @@ "log.level": "LOG", "log.offset": 897, "message": "autovacuum launcher started", - "postgresql.log.core_id": "1", + "postgresql.log.core_id": 1, "postgresql.log.timestamp": "2017-07-31 13:36:43.557", "postgresql.log.timezone": "EST", "process.pid": 835 @@ -71,7 +71,7 @@ "log.level": "LOG", "log.offset": 967, "message": "checkpoints are occurring too frequently (25 seconds apart)", - "postgresql.log.core_id": "1", + "postgresql.log.core_id": 1, "postgresql.log.timestamp": "2017-07-31 13:36:44.227", "postgresql.log.timezone": "EST", "process.pid": 832 @@ -84,7 +84,7 @@ "log.level": "HINT", "log.offset": 1069, "message": "Consider increasing the configuration parameter \"max_wal_size\".", - "postgresql.log.core_id": "2", + "postgresql.log.core_id": 2, "postgresql.log.timestamp": "2017-07-31 13:46:02.670", "postgresql.log.timezone": "EST", "process.pid": 832 @@ -97,7 +97,7 @@ "log.level": "FATAL", "log.offset": 1176, "message": "the database system is starting up", - "postgresql.log.core_id": "1", + "postgresql.log.core_id": 1, "postgresql.log.database": "postgres", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", "postgresql.log.timezone": "EST", @@ -112,7 +112,7 @@ "log.level": "FATAL", "log.offset": 1273, "message": "the database system is starting up", - "postgresql.log.core_id": "1", + "postgresql.log.core_id": 1, "postgresql.log.database": "postgres", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", "postgresql.log.timezone": "EST", From b95871855bb3b4ef98c63d26cb2daa6eab09e921 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 14:27:09 -0500 Subject: [PATCH 07/11] Migrate the timezone field to ECS as well --- .../postgresql/log/ingest/pipeline.json | 2 +- ...-9.6-debian-with-slowlog.log-expected.json | 36 +++++++++---------- ...ostgresql-9.6-multi-core.log-expected.json | 16 ++++----- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 9427ff91a63..d149f4be897 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,7 +6,7 @@ "field": "message", "ignore_missing": true, "patterns": [ - "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:event.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index 5c3107f8929..db026e9fcd3 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -3,61 +3,61 @@ "@timestamp": "2017-07-31T13:36:42.585Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 0, "message": "database system was shut down at 2017-06-17 16:58:04 CEST", "postgresql.log.timestamp": "2017-07-31 13:36:42.585", - "postgresql.log.timezone": "CEST", "process.pid": 4974 }, { "@timestamp": "2017-07-31T13:36:42.605Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 100, "message": "MultiXact member wraparound protections are now enabled", "postgresql.log.timestamp": "2017-07-31 13:36:42.605", - "postgresql.log.timezone": "CEST", "process.pid": 4974 }, { "@timestamp": "2017-07-31T13:36:42.615Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 198, "message": "autovacuum launcher started", "postgresql.log.timestamp": "2017-07-31 13:36:42.615", - "postgresql.log.timezone": "CEST", "process.pid": 4978 }, { "@timestamp": "2017-07-31T13:36:42.616Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 268, "message": "database system is ready to accept connections", "postgresql.log.timestamp": "2017-07-31 13:36:42.616", - "postgresql.log.timezone": "CEST", "process.pid": 4973 }, { "@timestamp": "2017-07-31T13:36:42.956Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 357, "message": "incomplete startup packet", "postgresql.log.database": "unknown", "postgresql.log.timestamp": "2017-07-31 13:36:42.956", - "postgresql.log.timezone": "CEST", "process.pid": 4980, "user.name": "unknown" }, @@ -66,6 +66,7 @@ "event.dataset": "log", "event.duration": 37118000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" @@ -77,7 +78,6 @@ "postgresql.log.duration": 37.118, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", - "postgresql.log.timezone": "CEST", "process.pid": 4983, "user.name": "postgres" }, @@ -86,6 +86,7 @@ "event.dataset": "log", "event.duration": 2895000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" @@ -97,7 +98,6 @@ "postgresql.log.duration": 2.895, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:44.104", - "postgresql.log.timezone": "CEST", "process.pid": 4986, "user.name": "postgres" }, @@ -106,6 +106,7 @@ "event.dataset": "log", "event.duration": 2809000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" @@ -117,7 +118,6 @@ "postgresql.log.duration": 2.809, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.timestamp": "2017-07-31 13:36:44.642", - "postgresql.log.timezone": "CEST", "process.pid": 4989, "user.name": "postgres" }, @@ -125,13 +125,13 @@ "@timestamp": "2017-07-31T13:39:16.249Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "FATAL", "log.offset": 1727, "message": "database \"users\" does not exist", "postgresql.log.database": "users", "postgresql.log.timestamp": "2017-07-31 13:39:16.249", - "postgresql.log.timezone": "CEST", "process.pid": 5407, "user.name": "postgres" }, @@ -139,13 +139,13 @@ "@timestamp": "2017-07-31T13:39:17.945Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "FATAL", "log.offset": 1818, "message": "database \"user\" does not exist", "postgresql.log.database": "user", "postgresql.log.timestamp": "2017-07-31 13:39:17.945", - "postgresql.log.timezone": "CEST", "process.pid": 5500, "user.name": "postgres" }, @@ -154,6 +154,7 @@ "event.dataset": "log", "event.duration": 37598000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" @@ -165,7 +166,6 @@ "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-07-31 13:39:21.025", - "postgresql.log.timezone": "CEST", "process.pid": 5404, "user.name": "postgres" }, @@ -174,6 +174,7 @@ "event.dataset": "log", "event.duration": 9482000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 2620, @@ -182,7 +183,6 @@ "postgresql.log.duration": 9.482, "postgresql.log.query": "select * from clients;", "postgresql.log.timestamp": "2017-07-31 13:39:31.619", - "postgresql.log.timezone": "CEST", "process.pid": 5502, "user.name": "postgres" }, @@ -191,6 +191,7 @@ "event.dataset": "log", "event.duration": 765000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 2733, @@ -199,7 +200,6 @@ "postgresql.log.duration": 0.765, "postgresql.log.query": "select id from clients;", "postgresql.log.timestamp": "2017-07-31 13:39:40.147", - "postgresql.log.timezone": "CEST", "process.pid": 5502, "user.name": "postgres" }, @@ -208,6 +208,7 @@ "event.dataset": "log", "event.duration": 26082001, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" @@ -219,7 +220,6 @@ "postgresql.log.duration": 26.082, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-07-31 13:40:54.310", - "postgresql.log.timezone": "CEST", "process.pid": 5502, "user.name": "postgres" }, @@ -228,6 +228,7 @@ "event.dataset": "log", "event.duration": 36161999, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 3559, @@ -236,7 +237,6 @@ "postgresql.log.duration": 36.162, "postgresql.log.query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.timestamp": "2017-07-31 13:43:22.645", - "postgresql.log.timezone": "CEST", "process.pid": 5502, "user.name": "postgres" }, @@ -245,6 +245,7 @@ "event.dataset": "log", "event.duration": 10540000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 3751, @@ -253,7 +254,6 @@ "postgresql.log.duration": 10.54, "postgresql.log.query": "insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", - "postgresql.log.timezone": "CEST", "process.pid": 5502, "user.name": "postgres" }, @@ -262,6 +262,7 @@ "event.dataset": "log", "event.duration": 5156000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 3908, @@ -270,7 +271,6 @@ "postgresql.log.duration": 5.156, "postgresql.log.query": "insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", - "postgresql.log.timezone": "CEST", "process.pid": 5502, "user.name": "postgres" }, @@ -279,6 +279,7 @@ "event.dataset": "log", "event.duration": 25871000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 4069, @@ -287,7 +288,6 @@ "postgresql.log.duration": 25.871, "postgresql.log.query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", - "postgresql.log.timezone": "CEST", "process.pid": 5502, "user.name": "postgres" } diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json index 9c419cb3242..c4c4d1b5536 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json @@ -3,6 +3,7 @@ "@timestamp": "2017-04-03T22:32:14.322Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "LOG", "log.offset": 0, @@ -10,7 +11,6 @@ "postgresql.log.core_id": 1, "postgresql.log.database": "unknown", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", - "postgresql.log.timezone": "CEST", "process.pid": 12975, "user.name": "unknown" }, @@ -18,6 +18,7 @@ "@timestamp": "2017-04-03T22:32:14.322Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.level": "FATAL", "log.offset": 91, @@ -25,7 +26,6 @@ "postgresql.log.core_id": 1, "postgresql.log.database": "user", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", - "postgresql.log.timezone": "CEST", "process.pid": 5404, "user.name": "postgres" }, @@ -34,6 +34,7 @@ "event.dataset": "log", "event.duration": 37598000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" @@ -46,7 +47,6 @@ "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.timestamp": "2017-04-03 22:35:22.389", - "postgresql.log.timezone": "CEST", "process.pid": 5404, "user.name": "postgres" }, @@ -54,45 +54,46 @@ "@timestamp": "2017-07-31T13:36:43.557Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", "log.level": "LOG", "log.offset": 897, "message": "autovacuum launcher started", "postgresql.log.core_id": 1, "postgresql.log.timestamp": "2017-07-31 13:36:43.557", - "postgresql.log.timezone": "EST", "process.pid": 835 }, { "@timestamp": "2017-07-31T13:36:44.227Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", "log.level": "LOG", "log.offset": 967, "message": "checkpoints are occurring too frequently (25 seconds apart)", "postgresql.log.core_id": 1, "postgresql.log.timestamp": "2017-07-31 13:36:44.227", - "postgresql.log.timezone": "EST", "process.pid": 832 }, { "@timestamp": "2017-07-31T13:46:02.670Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", "log.level": "HINT", "log.offset": 1069, "message": "Consider increasing the configuration parameter \"max_wal_size\".", "postgresql.log.core_id": 2, "postgresql.log.timestamp": "2017-07-31 13:46:02.670", - "postgresql.log.timezone": "EST", "process.pid": 832 }, { "@timestamp": "2017-07-31T13:46:23.016Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", "log.level": "FATAL", "log.offset": 1176, @@ -100,7 +101,6 @@ "postgresql.log.core_id": 1, "postgresql.log.database": "postgres", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", - "postgresql.log.timezone": "EST", "process.pid": 768, "user.name": "postgres" }, @@ -108,6 +108,7 @@ "@timestamp": "2017-07-31T13:46:55.637Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", "log.level": "FATAL", "log.offset": 1273, @@ -115,7 +116,6 @@ "postgresql.log.core_id": 1, "postgresql.log.database": "postgres", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", - "postgresql.log.timezone": "EST", "process.pid": 771, "user.name": "postgres" } From 6f4f8ba3816bb60da2daa3db9dbfd1895ad1ef22 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 16:10:35 -0500 Subject: [PATCH 08/11] Alias pgsql migrated fields, include the new migration attribute --- filebeat/docs/fields.asciidoc | 57 +++++++++---------- filebeat/module/postgresql/fields.go | 2 +- .../module/postgresql/log/_meta/fields.yml | 38 +++++++------ 3 files changed, 49 insertions(+), 48 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index f116680db01..e1ac0b5811f 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -9892,88 +9892,87 @@ The timestamp from the log line. -- -*`postgresql.log.timezone`*:: +*`postgresql.log.core_id`*:: + -- -The timezone of timestamp. +type: long + +Core id -- -*`postgresql.log.thread_id`*:: +*`postgresql.log.database`*:: + -- -type: long - -Process id +example: mydb +Name of database -- -*`postgresql.log.core_id`*:: +*`postgresql.log.duration`*:: + -- -type: long +type: float -Core id +example: 30.0 +Duration of a query. -- -*`postgresql.log.user`*:: +*`postgresql.log.query`*:: + -- -example: admin +example: SELECT * FROM users; -Name of user +Query statement. -- -*`postgresql.log.database`*:: +*`postgresql.log.timezone`*:: + -- -example: mydb +type: alias -Name of database +alias to: event.timezone -- -*`postgresql.log.level`*:: +*`postgresql.log.thread_id`*:: + -- -example: FATAL +type: alias -The log level. +alias to: process.pid -- -*`postgresql.log.duration`*:: +*`postgresql.log.user`*:: + -- -type: float - -example: 30.0 +type: alias -Duration of a query. +alias to: user.name -- -*`postgresql.log.query`*:: +*`postgresql.log.level`*:: + -- -example: SELECT * FROM users; +type: alias -Query statement. +alias to: log.level -- *`postgresql.log.message`*:: + -- -type: text - -The logged message. +type: alias +alias to: message -- diff --git a/filebeat/module/postgresql/fields.go b/filebeat/module/postgresql/fields.go index 826ba953f82..559ee7adfb9 100644 --- a/filebeat/module/postgresql/fields.go +++ b/filebeat/module/postgresql/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJysk8Fq3DAQhu9+ip89FrIEenOhENLsadMmzd6DshprRSWNI8kl26cv8sa1o7VNHTJHifm/j5HmAr/oWKLmEJWn8GwKIOpoqMTq7nT4cL9dFYCksPe6jppdia8FANyybAyhYo9a+KCdQjwQ+j4YVqi0obAugHBgHx/37CqtSkTfUAFUmowMZZt3AScsZTap4rGmEspzU7+ejNicatPmofJsM5HWIdUQOcQaVoOgc+Ys9y15bgxd5RpDlagthShs/eZ2Fp9qd6C+tTdJeKMdrSdZf9jR+1CpE1z12AnIwZOQj1pmQacpG3ZqGf7O855CQBbY4fbs6eNg1+xpitQE8lkDvQhbtxskpNVuNQM7Q30Xth3nWWzHkyKKJxHy1+qZ9iif3oUcTf63G/SbzCRzc7W72i6C7rpvmXLHv4xsvEjNo49YGRZx0ufz5fpykc63V1aag8BzQ/44LtVeTXIfbrY31zt8wubnj9v2DcOXRR73KR4hikiWXBx3sBSCUPkPOM0l0ks+lv9YZMNKkeyC18XfAAAA///JvYrW" + return "eJyck0FP3DAQhe/5FU97rESE1Fsq9ULhBG0p3NGwmThW7YzxOKjbX185AXbJJivSOcZ67/sycc7wm3cVgmgykfXJFUCyyXGFzc/x4d3t9aYAatZttCFZ6Sp8LQDgRureMRqJCBTVdgapZexzcGLQWMdaFoC2EtPDVrrGmgop9lwAjWVXazX0naEjzxObPGkXuIKJ0oeXJzM241wNfWii+InI4JDnEHmIdWIOio6ZJ7nvyafW8DpTjUOVZD1rIh/enZ7E57lveR/dm2S8sx2Xs6ytRH6w9aRsfH0nnVmncCGRMWl7JdWU6JGUJyH+Qz4MN87v6sfNCd4R7Tt5hjTzzW/YPlKOz75h44TSotDn8/J8ldC3F1aWIjz1HHfzWx+OFrl3l9eXF/f4hKtfP27QK0f9ssrjNtdDEyX23KW9w+J1+yvd9MOMGyJnSScngVJbgZ9z9ULYWzOu4u1Xn+G2kaleunzL4BBly6plOEp+hJrXuRKYI2VO/wfO8TO7lTwnppzLfYTnWZXM2m85n5ry/gUAAP//D0KTig==" } diff --git a/filebeat/module/postgresql/log/_meta/fields.yml b/filebeat/module/postgresql/log/_meta/fields.yml index 4e5a451eace..92b48ffb8c4 100644 --- a/filebeat/module/postgresql/log/_meta/fields.yml +++ b/filebeat/module/postgresql/log/_meta/fields.yml @@ -6,29 +6,14 @@ - name: timestamp description: > The timestamp from the log line. - - name: timezone - description: > - The timezone of timestamp. - - name: thread_id - type: long - description: > - Process id - name: core_id type: long description: > Core id - - name: user - example: "admin" - description: - Name of user - name: database example: "mydb" description: Name of database - - name: level - example: "FATAL" - description: - The log level. - name: duration type: float example: "30.0" @@ -38,7 +23,24 @@ example: "SELECT * FROM users;" description: Query statement. + + - name: timezone + type: alias + path: event.timezone + migration: true + - name: thread_id + type: alias + path: process.pid + migration: true + - name: user + type: alias + path: user.name + migration: true + - name: level + type: alias + path: log.level + migration: true - name: message - type: text - description: > - The logged message. + type: alias + path: message + migration: true From 496440ea5da61ab30590778c85ee1267dda50728 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 16:18:41 -0500 Subject: [PATCH 09/11] Document field name transitions in ecs-migration.yml --- dev-tools/ecs-migration.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index f95542a780d..3003a499fa2 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -434,6 +434,28 @@ to: user_agent.original alias: true +## PostgreSQL module + +- from: postgresql.log.timezone + to: + alias: true + +- from: postgresql.log.thread_id + to: process.pid + alias: true + +- from: postgresql.log.user + to: user.name + alias: true + +- from: postgresql.log.level + to: log.level + alias: true + +- from: postgresql.log.message + to: message + alias: true + ## Redis module - from: redis.log.pid From 4823a8d9a065d0decaee562db19436e3f9ad8732 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 19 Dec 2018 16:19:27 -0500 Subject: [PATCH 10/11] Changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0b42e13e71b..6b6076f65d5 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -181,6 +181,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the - Use `log.source.address` instead of `log.source.ip` for network input sources. {pull}9487[9487] - Rename many `redis.log.*` fields to map to ECS. {pull}9315[9315] - Rename many `icinga.*` fields to map to ECS. {pull}9294[9294] +- Rename many `postgresql.log.*` fields to map to ECS. {pull}9303[9303] *Metricbeat* From a350d413d4bffb2a9b3df0412d63c9f7f38e0f56 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 20 Dec 2018 09:23:33 -0500 Subject: [PATCH 11/11] Fix alias in ecs-migration.yml --- dev-tools/ecs-migration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 3003a499fa2..d7d0cd80931 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -437,7 +437,7 @@ ## PostgreSQL module - from: postgresql.log.timezone - to: + to: event.timezone alias: true - from: postgresql.log.thread_id