From a0a2f5713a53507ec6c39b00339f19afb2efe7e2 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 21 Jan 2019 18:53:51 +0100 Subject: [PATCH 1/3] Support percona extended slow log Fix #6665 Co-authored-by: Hagen Montag --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 156 ++++++++++++++++++ filebeat/module/mysql/fields.go | 2 +- .../module/mysql/slowlog/_meta/fields.yml | 71 ++++++++ .../module/mysql/slowlog/ingest/pipeline.json | 2 +- .../test/mariadb-10.2.12.log-expected.json | 2 +- .../test/percona-ubuntu-5.7.19-innodb.log | 36 ++++ ...ona-ubuntu-5.7.19-innodb.log-expected.json | 90 ++++++++++ .../slowlog/test/percona-ubuntu-5.7.19.log | 121 ++++++++++++++ 9 files changed, 478 insertions(+), 3 deletions(-) create mode 100644 filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log create mode 100644 filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json create mode 100644 filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19.log diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b102d2676c00..007f0bdf7d9c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -151,6 +151,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - HAProxy module now populates `event.duration` and `http.response.bytes` (ECS). {pull}10143[10143] - Teach elasticsearch/audit fileset to parse out some more fields. {issue}10134[10134] {pull}10137[10137] - Add convert_timezone to nginx module. {issue}9839[9839] {pull}10148[10148] +- Add support for Percona in the `slowlog` fileset of `mysql` module. {issue}6665[6665] {pull}10227[10227] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 952a85c6e8a0..5cf55cd7b979 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -6443,6 +6443,16 @@ type: long The number of rows modified by the query. +-- + +*`mysql.slowlog.bytes_sent`*:: ++ +-- +type: long + +The size of the query result. + + -- *`mysql.slowlog.query`*:: @@ -6480,6 +6490,26 @@ type: keyword Current authenticated user, used to determine access privileges. Can differ from the value for user. +-- + +*`mysql.slowlog.last_errno`*:: ++ +-- +type: keyword + +Last SQL error seen. + + +-- + +*`mysql.slowlog.killed`*:: ++ +-- +type: keyword + +Code of the reason if the query was killed. + + -- *`mysql.slowlog.query_cache_hit`*:: @@ -6510,6 +6540,35 @@ type: boolean Whether the query needed temporary tables on disk. +-- + +*`mysql.slowlog.tmp_tables`*:: ++ +-- +type: long + +Number of temporary tables created for this query + + +-- + +*`mysql.slowlog.tmp_disk_tables`*:: ++ +-- +type: long + +Number of temporary tables created on disk for this query. + + +-- + +*`mysql.slowlog.tmp_table_sizes`*:: ++ +-- +type: long + +Size of temporary tables created for this query. + -- *`mysql.slowlog.filesort`*:: @@ -6570,6 +6629,103 @@ type: long Number of merge passes executed for the query. +-- + +*`mysql.slowlog.log_slow_rate_type`*:: ++ +-- +type: keyword + +Type of slow log rate limit, it can be `session` if the rate limit is applied per session, or `query` if it applies per query. + + +-- + +*`mysql.slowlog.log_slow_rate_limit`*:: ++ +-- +type: keyword + +Slow log rate limit, a value of 100 means that one in a hundred queries or sessions are being logged. + + +-- + +[float] +== innodb fields + +Contains fields relative to InnoDB engine + + + +*`mysql.slowlog.innodb.trx_id`*:: ++ +-- +type: keyword + +Transaction ID + + +-- + +*`mysql.slowlog.innodb.io_r_ops`*:: ++ +-- +type: long + +Number of page read operations. + + +-- + +*`mysql.slowlog.innodb.io_r_bytes`*:: ++ +-- +type: long + +Bytes read during page read operations. + + +-- + +*`mysql.slowlog.innodb.io_r_wait.sec`*:: ++ +-- +type: long + +How long it took to read all needed data from storage. + + +-- + +*`mysql.slowlog.innodb.rec_lock_wait.sec`*:: ++ +-- +type: long + +How long the query waited for locks. + + +-- + +*`mysql.slowlog.innodb.queue_wait.sec`*:: ++ +-- +type: long + +How long the query waited to enter the InnoDB queue and to be executed once in the queue. + + +-- + +*`mysql.slowlog.innodb.pages_distinct`*:: ++ +-- +type: long + +Approximated count of pages accessed to execute the query. + + -- *`mysql.slowlog.user`*:: diff --git a/filebeat/module/mysql/fields.go b/filebeat/module/mysql/fields.go index 8825a56ce115..c105c1a9666f 100644 --- a/filebeat/module/mysql/fields.go +++ b/filebeat/module/mysql/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEV82O8zYMvOcpiO/UAvv5AXLoZYsCBfoVKFqgR4ORaJuNLHpFOdn06Qspv3XsbJKuUR1ysMCZ4Ugkla+wpt0S2p2+uQVA5OhoCV++7X7/7ZcvCwBLagJ3kcUv4YcFAMA3sb0jqCRAh0HZ1xAbghwCTmqo2JEWCwBtJMTSiK+4XkIMPS0AKiZndZmhvoLHls70acVdR0uog/Td4cuIhrR+ykBQBWmnBKR1yXfJGZtAaEu2p50jtxNfX3ycoE/rj4bAiPdk0i5IOKDCzz9mf5Kst57CrriipxAkXFFfpv0B96v4iOz1kODQh4yf3NDiImxoxkeGnJWhY9TBToexORxeMRXdch1wr/9wAa6ZHW3IPcjqpC7G4u7ha0kVa3o0z9GoKb4jlzrZOqlnPOnE8NBB5xtZRm6pUDKjLlROMA52biiEQy1EieggIZ+vPkSR9QuwByUj3uoLoALuKVLz6IR9BN+3KwrF+P0Qs55FL7bS+whSDTVvkSPZUw0nARAFVgS4QXa4clQkhCvUDbqegPU/5Rtkq6WSH2Y02p7uTHXPl1JN6BAo9sGThdVurEtdqaF3bNnTeG/4FEVq0D8gCKuKTJxTUCuWK75LUd56nDBX7g3Y/6EVq2moxVHaNe22Eoag96SZMWHbUNjX2Dlv2KICvZPpI9lxE0wfAvlY9krh83S97lEB+9iQj2ww1XvieEm/NlW7pUgh3XpAY0gVusAbdlSTFvCKHixXFYVzP94Xf2oaCeiczo0ubNA0VDY8XuorEUfoH0vtz4ZiQxePD8gk2eqG47jLse3KmNra5+tAiNR2EjDNgkSRlRxNDqTiNvRRhZ30leJLy7qe0y9PZJO6f8tWkHTkuh5XmB+dEmY4yCMySBe55b9zDZ9MvK1mPrtuqwL0Fjg+Z2UXWALHXfnWUz/LjTwyQGY4q061e0xswtjeuTLNqjlkJfBDjSSKrOvg4PEp8tHMyPr+Ep5PXwK/rQy+8wLsLb2TwjZ1/ZO5KVi/H5feUqip7FCVhuPt2bH+62mkZ3DYg58mztSftFF5kwNoeh7nMZCin5jEjeh4L5mmU+mDocJKi1fnfw8ld88RXsUNyf4JAAD//zJjbbQ=" + return "eJzEmN9v3DYMx9/zVxDdywakTvcwDMjDgLXFsALtgKEF9ujyLNrWThZdkc7l+tcPkn0/erYvl+Ru9UOAnC1+P6QkiuJLWNL6Fpq1fHFXAGrV0S28+LD++Pf7F1cAhqQItlXL/hZ+uwIA+MCmcwQlB2gxiPUVaE2QhoDjCkrrSLIrAKk5aF6wL211Cxo6ugIoLTkjt8nUS/DY0E4+Prpu6RaqwF07/DLBEJ8/kiEoAzdzAPHZ19vX1DoQmtya7ZuNtmNf7f04Ix+fTzVBwd5TEd8Ch8EqvHub4hOxvnQU1tlInkLgMJLed/sB7TfsFa2XwcHDOCT7MRqS7Q07DMZDAdmRobMoB29a1HqYvGxudGOrgD3/sADGyo7uyD1S1XGVTY07Ra8hEazosX5OjprT22iJ45Xj6oIzHRUeNdFpReZqG8qEiskolI5RD94cIYRhLygrOoiWd0sflHl5DdaDUMHeyDWgAPYSMXm0bL2C75oFhWx6fXCxvAgvNtx5BS4PmVdolcx2D0cAUIYFAd6hdbhwlEULI6t36DoCK8/yN/BKciF/6NFkejrR1V4vuhqtQyDtgicDi/VUlhrR0D021tN0bjgLkRToHwGEZUmFXhKoYWNLexLRYq109hkT+5XS0tyuykDSOZ1GSF88QSQmjyOefYfTQIqaGpyUXdJ6xeHQ6CluJpuwqin023znN6xQgO6p6JTMdBCKLgTymndC4Xxcb3qrgJ3W5NUWGFNO1LiOf01MOIaUQtx4gEVBItAGe2cdVSQZvEEPxpYlhd2R0OefmLeioZl0iqI5heD5fM68R1HY1R1C5KfFl9a5mV37tCiy2e6SQCjswZbfZHIZNHc8R07FAoua8tpOb+QFsyP0jyP8pyataa8YhCSSyGo7s521aXONx8z5ORCUmpYDxrM5SiSSzYoLJOzu6KGMt+XL2efGyvKS8fJEJtJ9iy3Acf3L8gHCwzx1Yk6GEdhf2/NhRFIEwl3BYGUiHe9zRepnwT2FbYjWAeND0xtPoadBjhg/bg6006I3TZbudhwusD83loFbtY39ms6p7d44TnO5XXCcCtAbsPq0HdIGy8HqOv/SUXeRRLNRgKSwo44TvXFsJrCdc3ksCS+BFY0PqS9KJK4hgpuK/6G6KPH9y/ZyfNH4cTL40TNYb+ieBFaxstkGNw6Wn6bRGwoV5S2KXCD7JOPQG99WVXO9EPjmjlfl0bU8oFIeQc5YAa7blHg2N2WIGuBsY/U67p64BhYEn4VELPvPmwpi99nIpBXAtnXxetBSrHfSyGvgAJ+Tl8mI1eErSV+d7P6U5jP8/zjlNw61Ipfw86tX0BB6Aa1RgT3F6ytC3XkTyCRuO1osALx1XAADwYLi/dZxVc1WWz9ArdrK7c3NarXKWgoFe8wKbm4MFzfD/y+Fwh2Fm1+yX2+MxcqzqC3kJgWI7pW8IZPV2rjpO4v3bBaT4TvsupwQvMPuSyCHamOFxPDOe377GshX1h+u16kOzD6lhvvx/QgemOgTeOPzKaAX7JuS797OIljOQ87teGKPZIITCXYZocUqVeYGuKX+8ifZcaR0nb4A1Otot2cxXUi9mMfDrdDqRB/qHHx/pm3qq5g3lHnZ1+NoAJ3bHAMGFfvrnigHrGgeN1CRp9bZ/4E82TuL6kfimaqC70OnDOR1uGUM27gvUmJF1bf6tscX+2Lc6YuP9RvL3ZF5iItMYnmo1hfjo+T5bv7etoHvbZOK6GLT0kyyQ99g8Lh3aOoonkyks/2O+fZP6jrE0U9o/NQs04fevJxwFwrKDDc4KsVOkbSHh8GJgqNxc2JX/wUAAP//ri9Ugg==" } diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml index 38147609ba9a..a64d915d420c 100644 --- a/filebeat/module/mysql/slowlog/_meta/fields.yml +++ b/filebeat/module/mysql/slowlog/_meta/fields.yml @@ -24,6 +24,10 @@ type: long description: > The number of rows modified by the query. + - name: bytes_sent + type: long + description: > + The size of the query result. - name: query description: > The slow query. @@ -39,6 +43,14 @@ type: keyword description: > Current authenticated user, used to determine access privileges. Can differ from the value for user. + - name: last_errno + type: keyword + description: > + Last SQL error seen. + - name: killed + type: keyword + description: > + Code of the reason if the query was killed. - name: query_cache_hit type: boolean @@ -52,6 +64,18 @@ type: boolean description: > Whether the query needed temporary tables on disk. + - name: tmp_tables + type: long + description: > + Number of temporary tables created for this query + - name: tmp_disk_tables + type: long + description: > + Number of temporary tables created on disk for this query. + - name: tmp_table_sizes + type: long + description: + Size of temporary tables created for this query. - name: filesort type: boolean description: > @@ -76,6 +100,52 @@ type: long description: > Number of merge passes executed for the query. + - name: log_slow_rate_type + type: keyword + description: > + Type of slow log rate limit, it can be `session` if the rate limit + is applied per session, or `query` if it applies per query. + - name: log_slow_rate_limit + type: keyword + description: > + Slow log rate limit, a value of 100 means that one in a hundred queries + or sessions are being logged. + + # https://www.percona.com/doc/percona-server/5.7/diagnostics/slow_extended.html + - name: innodb + type: group + description: > + Contains fields relative to InnoDB engine + fields: + - name: trx_id + type: keyword + description: > + Transaction ID + - name: io_r_ops + type: long + description: > + Number of page read operations. + - name: io_r_bytes + type: long + description: > + Bytes read during page read operations. + - name: io_r_wait.sec + type: long + description: > + How long it took to read all needed data from storage. + - name: rec_lock_wait.sec + type: long + description: > + How long the query waited for locks. + - name: queue_wait.sec + type: long + description: > + How long the query waited to enter the InnoDB queue and to be executed once + in the queue. + - name: pages_distinct + type: long + description: > + Approximated count of pages accessed to execute the query. - name: user type: alias @@ -89,3 +159,4 @@ type: alias path: source.ip migration: true + diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index 9d517d30535e..4fb3c0e76e23 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -4,7 +4,7 @@ "grok": { "field": "message", "patterns":[ - "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\](\\s*Id:\\s* %{NUMBER:mysql.thread_id:long})?\n(# Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE}Schema: %{WORD:mysql.slowlog.schema}?%{METRICSPACE}QC_hit: %{WORD:mysql.slowlog.query_cache_hit}\n)?%{METRICSPACE}Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE}Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE}Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE}Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE}(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}" + "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}" ], "pattern_definitions" : { "GREEDYMULTILINE": "(.|\n)*", diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json index 6517ef123f20..1bab861381fd 100644 --- a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json @@ -42,7 +42,7 @@ "mysql.slowlog.full_join": false, "mysql.slowlog.full_scan": true, "mysql.slowlog.lock_time.sec": 0.0, - "mysql.slowlog.merge_passes": "0", + "mysql.slowlog.merge_passes": 0, "mysql.slowlog.priority_queue": false, "mysql.slowlog.query": "call PROC('blah');", "mysql.slowlog.query_cache_hit": false, diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log new file mode 100644 index 000000000000..36bdace36c96 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log @@ -0,0 +1,36 @@ +# Time: 2019-01-21T06:36:03.868128Z +# User@Host: exporter[exporter] @ localhost [] Id: 14367293 +# Schema: Last_errno: 1370 Killed: 0 +# Query_time: 0.050365 Lock_time: 0.010733 Rows_sent: 16 Rows_examined: 3146 Rows_affected: 0 +# Bytes_sent: 1362 Tmp_tables: 376 Tmp_disk_tables: 71 Tmp_table_sizes: 606208 +# QC_Hit: No Full_scan: Yes Full_join: Yes Tmp_table: Yes Tmp_table_on_disk: Yes +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# No InnoDB statistics available for this query +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1548052563; +SELECT t.table_schema, t.table_name, column_name, `auto_increment`, + pow(2, case data_type + when 'tinyint' then 7 + when 'smallint' then 15 + when 'mediumint' then 23 + when 'int' then 31 + when 'bigint' then 63 + end+(column_type like '% unsigned'))-1 as max_int + FROM information_schema.tables t + JOIN information_schema.columns c + ON BINARY t.table_schema = c.table_schema AND BINARY t.table_name = c.table_name + WHERE c.extra = 'auto_increment' AND t.auto_increment IS NOT NULL; +# Time: 2019-01-21T09:15:36.487710Z +# User@Host: test[test] @ [192.168.123.123] Id: 14370752 +# Schema: test Last_errno: 3170 Killed: 0 +# Query_time: 153.883481 Lock_time: 0.024022 Rows_sent: 0 Rows_examined: 120309968 Rows_affected: 19198 +# Bytes_sent: 62 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# InnoDB_trx_id: 69BB9C7F9 +# QC_Hit: No Full_scan: No Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# InnoDB_IO_r_ops: 9744 InnoDB_IO_r_bytes: 79822848 InnoDB_IO_r_wait: 0.883446 +# InnoDB_rec_lock_wait: 0.003038 InnoDB_queue_wait: 0.000000 +# InnoDB_pages_distinct: 64872 +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1548062136; +UPDATE test SET test.state = 'NOT_RELEVANT', modified = now() WHERE test.id IN (26328833, 390, 149386, 152268, 160997, 165304, 168524, 184105, 193022, 194533, 194862, 196469, 196487, 246398, 256594, 260566, 261862, 262342, 263701, 264166, 264607, 267671, 274879, 276704, 280964, 284366, 289323, 289843, 290004, 298999, 301213, 303494, 307920, 311905, 316311, 318404, 330846, 340751, 341433, 357191, 369184, 376876, 378360, 378492, 379470, 382131, 384077, 388368, 396815, 396881, 398272, 398950, 399589, 401299, 408787, 411293, 419109, 425953, 427659, 433183, 437030, 438332, 438386, 447037, 454231, 455257, 455344, 456385, 460420, 460425, 461252, 462338, 462531, 462684, 463104, 463395, 471073, 480069, 480078, 482399, 485205, 487971, 497191, 500261, 501855, 517585, 519310, 519654, 522575, 538425, 543560, 562315, 573934, 583466, 583490, 583502, 597605, 600875, 601546, 603879, 604467, 604619, 757786, 797285, 799155, 802905, 806268, 806798, 811974, 819684, 822629, 826406, 837733, 840128, 840131, 840251, 840277, 840302, 842966, 844294, 844300, 847837, 852503, 854272, 854299, 862983, 881405, 881461, 881467, 881560, 881908, 882435, 882453, 882651, 882711, 882811, 888265, 888286, 914091, 916288, 916316, 917708, 918238, 918887, 919222, 926607, 976977, 977010, 977067, 977131, 977185, 988249, 988276, 988336, 988360, 988504, 990994); diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json new file mode 100644 index 000000000000..a45e9be1c781 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json @@ -0,0 +1,90 @@ +[ + { + "@timestamp": "2019-01-21T06:36:03.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.slowlog", + "event.duration": 50365000, + "event.module": "mysql", + "fileset.name": "slowlog", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 36, + "mysql.slowlog.bytes_sent": 1362, + "mysql.slowlog.current_user": "exporter", + "mysql.slowlog.filesort": false, + "mysql.slowlog.filesort_on_disk": false, + "mysql.slowlog.full_join": true, + "mysql.slowlog.full_scan": true, + "mysql.slowlog.killed": 0, + "mysql.slowlog.last_errno": 1370, + "mysql.slowlog.lock_time.sec": 0.010733, + "mysql.slowlog.log_slow_rate_limit": 100, + "mysql.slowlog.log_slow_rate_type": "query", + "mysql.slowlog.merge_passes": 0, + "mysql.slowlog.query": "SELECT t.table_schema, t.table_name, column_name, `auto_increment`,\n pow(2, case data_type\n when 'tinyint' then 7\n when 'smallint' then 15\n when 'mediumint' then 23\n when 'int' then 31\n when 'bigint' then 63\n end+(column_type like '% unsigned'))-1 as max_int\n FROM information_schema.tables t\n JOIN information_schema.columns c\n ON BINARY t.table_schema = c.table_schema AND BINARY t.table_name = c.table_name\n WHERE c.extra = 'auto_increment' AND t.auto_increment IS NOT NULL;", + "mysql.slowlog.query_cache_hit": false, + "mysql.slowlog.query_time.sec": 0.050365, + "mysql.slowlog.rows_affected": 0, + "mysql.slowlog.rows_examined": 3146, + "mysql.slowlog.rows_sent": 16, + "mysql.slowlog.tmp_disk_tables": "71", + "mysql.slowlog.tmp_table": true, + "mysql.slowlog.tmp_table_on_disk": true, + "mysql.slowlog.tmp_table_sizes": 606208, + "mysql.slowlog.tmp_tables": 376, + "mysql.thread_id": 14367293, + "service.type": "mysql", + "source.domain": "localhost", + "user.name": "exporter" + }, + { + "@timestamp": "2019-01-21T09:15:36.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.slowlog", + "event.duration": 153883488000, + "event.module": "mysql", + "fileset.name": "slowlog", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 1311, + "mysql.slowlog.bytes_sent": 62, + "mysql.slowlog.current_user": "test", + "mysql.slowlog.filesort": false, + "mysql.slowlog.filesort_on_disk": false, + "mysql.slowlog.full_join": false, + "mysql.slowlog.full_scan": false, + "mysql.slowlog.innodb.io_r_bytes": 79822848, + "mysql.slowlog.innodb.io_r_ops": 9744, + "mysql.slowlog.innodb.io_r_wait.sec": 0.883446, + "mysql.slowlog.innodb.pages_distinct": 64872, + "mysql.slowlog.innodb.queue_wait.sec": 0.0, + "mysql.slowlog.innodb.rec_lock_wait.sec": 0.003038, + "mysql.slowlog.innodb.trx_id": "69BB9C7F9", + "mysql.slowlog.killed": 0, + "mysql.slowlog.last_errno": 3170, + "mysql.slowlog.lock_time.sec": 0.024022, + "mysql.slowlog.log_slow_rate_limit": 100, + "mysql.slowlog.log_slow_rate_type": "query", + "mysql.slowlog.merge_passes": 0, + "mysql.slowlog.query": "UPDATE test SET test.state = 'NOT_RELEVANT', modified = now() WHERE test.id IN (26328833, 390, 149386, 152268, 160997, 165304, 168524, 184105, 193022, 194533, 194862, 196469, 196487, 246398, 256594, 260566, 261862, 262342, 263701, 264166, 264607, 267671, 274879, 276704, 280964, 284366, 289323, 289843, 290004, 298999, 301213, 303494, 307920, 311905, 316311, 318404, 330846, 340751, 341433, 357191, 369184, 376876, 378360, 378492, 379470, 382131, 384077, 388368, 396815, 396881, 398272, 398950, 399589, 401299, 408787, 411293, 419109, 425953, 427659, 433183, 437030, 438332, 438386, 447037, 454231, 455257, 455344, 456385, 460420, 460425, 461252, 462338, 462531, 462684, 463104, 463395, 471073, 480069, 480078, 482399, 485205, 487971, 497191, 500261, 501855, 517585, 519310, 519654, 522575, 538425, 543560, 562315, 573934, 583466, 583490, 583502, 597605, 600875, 601546, 603879, 604467, 604619, 757786, 797285, 799155, 802905, 806268, 806798, 811974, 819684, 822629, 826406, 837733, 840128, 840131, 840251, 840277, 840302, 842966, 844294, 844300, 847837, 852503, 854272, 854299, 862983, 881405, 881461, 881467, 881560, 881908, 882435, 882453, 882651, 882711, 882811, 888265, 888286, 914091, 916288, 916316, 917708, 918238, 918887, 919222, 926607, 976977, 977010, 977067, 977131, 977185, 988249, 988276, 988336, 988360, 988504, 990994);", + "mysql.slowlog.query_cache_hit": false, + "mysql.slowlog.query_time.sec": 153.88348, + "mysql.slowlog.rows_affected": 19198, + "mysql.slowlog.rows_examined": 120309968, + "mysql.slowlog.rows_sent": 0, + "mysql.slowlog.schema": "test", + "mysql.slowlog.tmp_disk_tables": "0", + "mysql.slowlog.tmp_table": false, + "mysql.slowlog.tmp_table_on_disk": false, + "mysql.slowlog.tmp_table_sizes": 0, + "mysql.slowlog.tmp_tables": 0, + "mysql.thread_id": 14370752, + "service.type": "mysql", + "source.ip": "192.168.123.123", + "user.name": "test" + } +] \ No newline at end of file diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19.log b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19.log new file mode 100644 index 000000000000..1572af7ff2c5 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19.log @@ -0,0 +1,121 @@ +# Time: 2018-11-16T06:25:56.646222Z +# User@Host: check[check] @ localhost [] Id: 1098148226 +# Schema: Last_errno: 0 Killed: 0 +# Query_time: 0.010569 Lock_time: 0.000067 Rows_sent: 1 Rows_examined: 928 Rows_affected: 0 +# Bytes_sent: 180 Tmp_tables: 1 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: Yes Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# No InnoDB statistics available for this query +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1542349556; +SHOW GLOBAL STATUS LIKE 'wsrep_local_state'; +# Time: 2018-11-16T13:02:59.046552Z +# User@Host: select[select] @ [192.168.123.123] Id: 1101779094 +# Schema: database Last_errno: 0 Killed: 0 +# Query_time: 0.036112 Lock_time: 0.000165 Rows_sent: 1 Rows_examined: 1 Rows_affected: 0 +# Bytes_sent: 529 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# QC_Hit: No Full_scan: No Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# InnoDB_IO_r_ops: 0 InnoDB_IO_r_bytes: 0 InnoDB_IO_r_wait: 0.000000 +# InnoDB_rec_lock_wait: 0.000000 InnoDB_queue_wait: 0.000000 +# InnoDB_pages_distinct: 3 +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1542373379; +select config.id as id, config.active as active from config where config.id='123456'; +# Time: 2019-01-21T06:33:10.615939Z +# User@Host: exporter[exporter] @ localhost [] Id: 14366748 +# Schema: Last_errno: 0 Killed: 0 +# Query_time: 0.023385 Lock_time: 0.000039 Rows_sent: 390 Rows_examined: 390 Rows_affected: 0 +# Bytes_sent: 20195 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# No InnoDB statistics available for this query +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1548052390; +SELECT EVENT_NAME, COUNT_STAR, SUM_TIMER_WAIT + FROM performance_schema.events_waits_summary_global_by_event_name; +# Time: 2019-01-21T06:34:30.932734Z +# User@Host: test[test] @ [192.168.123.123] Id: 14349788 +# Schema: test Last_errno: 0 Killed: 0 +# Query_time: 0.010278 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0 Rows_affected: 0 +# Bytes_sent: 11 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# InnoDB_trx_id: 69B884E82 +# QC_Hit: No Full_scan: No Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# No InnoDB statistics available for this query +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1548052470; +commit; +# Time: 2019-01-21T06:35:33.840616Z +# User@Host: exporter[exporter] @ localhost [] Id: 14367106 +# Schema: Last_errno: 1370 Killed: 0 +# Query_time: 0.014315 Lock_time: 0.000095 Rows_sent: 101 Rows_examined: 101 Rows_affected: 0 +# Bytes_sent: 7131 Tmp_tables: 111 Tmp_disk_tables: 24 Tmp_table_sizes: 114688 +# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: Yes Tmp_table_on_disk: Yes +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# No InnoDB statistics available for this query +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1548052533; +SELECT + TABLE_SCHEMA, + TABLE_NAME, + TABLE_TYPE, + ifnull(ENGINE, 'NONE') as ENGINE, + ifnull(VERSION, '0') as VERSION, + ifnull(ROW_FORMAT, 'NONE') as ROW_FORMAT, + ifnull(TABLE_ROWS, '0') as TABLE_ROWS, + ifnull(DATA_LENGTH, '0') as DATA_LENGTH, + ifnull(INDEX_LENGTH, '0') as INDEX_LENGTH, + ifnull(DATA_FREE, '0') as DATA_FREE, + ifnull(CREATE_OPTIONS, 'NONE') as CREATE_OPTIONS + FROM information_schema.tables + WHERE TABLE_SCHEMA = 'sys'; +# Time: 2019-01-21T06:36:03.868128Z +# User@Host: exporter[exporter] @ localhost [] Id: 14367293 +# Schema: Last_errno: 1370 Killed: 0 +# Query_time: 0.050365 Lock_time: 0.010733 Rows_sent: 16 Rows_examined: 3146 Rows_affected: 0 +# Bytes_sent: 1362 Tmp_tables: 376 Tmp_disk_tables: 71 Tmp_table_sizes: 606208 +# QC_Hit: No Full_scan: Yes Full_join: Yes Tmp_table: Yes Tmp_table_on_disk: Yes +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# No InnoDB statistics available for this query +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1548052563; +SELECT t.table_schema, t.table_name, column_name, `auto_increment`, + pow(2, case data_type + when 'tinyint' then 7 + when 'smallint' then 15 + when 'mediumint' then 23 + when 'int' then 31 + when 'bigint' then 63 + end+(column_type like '% unsigned'))-1 as max_int + FROM information_schema.tables t + JOIN information_schema.columns c + ON BINARY t.table_schema = c.table_schema AND BINARY t.table_name = c.table_name + WHERE c.extra = 'auto_increment' AND t.auto_increment IS NOT NULL; +# Time: 2019-01-21T06:36:40.896376Z +# User@Host: test[test] @ [192.168.123.123] Id: 14360213 +# Schema: test Last_errno: 0 Killed: 0 +# Query_time: 32.463767 Lock_time: 0.000084 Rows_sent: 267 Rows_examined: 267 Rows_affected: 0 +# Bytes_sent: 43805 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# QC_Hit: No Full_scan: No Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# InnoDB_IO_r_ops: 2 InnoDB_IO_r_bytes: 16384 InnoDB_IO_r_wait: 0.000213 +# InnoDB_rec_lock_wait: 0.000000 InnoDB_queue_wait: 0.000000 +# InnoDB_pages_distinct: 64832 +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1548052600; +select test.id as id, test.modified as mo, test.product as pr from test where (test.state in ('NOT_RELEVANT')) and test.last<='2019-01-21 06:36:08.432' and test.modified<='2019-01-07 06:36:08.432' limit 100000; +# Time: 2019-01-21T09:15:36.487710Z +# User@Host: test[test] @ [192.168.123.123] Id: 14370752 +# Schema: test Last_errno: 3170 Killed: 0 +# Query_time: 153.883481 Lock_time: 0.024022 Rows_sent: 0 Rows_examined: 120309968 Rows_affected: 19198 +# Bytes_sent: 62 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# InnoDB_trx_id: 69BB9C7F9 +# QC_Hit: No Full_scan: No Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# InnoDB_IO_r_ops: 9744 InnoDB_IO_r_bytes: 79822848 InnoDB_IO_r_wait: 0.883446 +# InnoDB_rec_lock_wait: 0.003038 InnoDB_queue_wait: 0.000000 +# InnoDB_pages_distinct: 64872 +# Log_slow_rate_type: query Log_slow_rate_limit: 100 +SET timestamp=1548062136; +UPDATE test SET test.state = 'NOT_RELEVANT', modified = now() WHERE test.id IN (26328833, 390, 149386, 152268, 160997, 165304, 168524, 184105, 193022, 194533, 194862, 196469, 196487, 246398, 256594, 260566, 261862, 262342, 263701, 264166, 264607, 267671, 274879, 276704, 280964, 284366, 289323, 289843, 290004, 298999, 301213, 303494, 307920, 311905, 316311, 318404, 330846, 340751, 341433, 357191, 369184, 376876, 378360, 378492, 379470, 382131, 384077, 388368, 396815, 396881, 398272, 398950, 399589, 401299, 408787, 411293, 419109, 425953, 427659, 433183, 437030, 438332, 438386, 447037, 454231, 455257, 455344, 456385, 460420, 460425, 461252, 462338, 462531, 462684, 463104, 463395, 471073, 480069, 480078, 482399, 485205, 487971, 497191, 500261, 501855, 517585, 519310, 519654, 522575, 538425, 543560, 562315, 573934, 583466, 583490, 583502, 597605, 600875, 601546, 603879, 604467, 604619, 757786, 797285, 799155, 802905, 806268, 806798, 811974, 819684, 822629, 826406, 837733, 840128, 840131, 840251, 840277, 840302, 842966, 844294, 844300, 847837, 852503, 854272, 854299, 862983, 881405, 881461, 881467, 881560, 881908, 882435, 882453, 882651, 882711, 882811, 888265, 888286, 914091, 916288, 916316, 917708, 918238, 918887, 919222, 926607, 976977, 977010, 977067, 977131, 977185, 988249, 988276, 988336, 988360, 988504, 990994); From 441c1efdef5042362f43d5520ad0c5d177a57ed3 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 22 Jan 2019 11:13:29 +0100 Subject: [PATCH 2/3] Add format to fields in bytes --- filebeat/docs/fields.asciidoc | 6 ++++++ filebeat/module/mysql/fields.go | 2 +- filebeat/module/mysql/slowlog/_meta/fields.yml | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 5cf55cd7b979..8245a0d45d26 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -6450,6 +6450,8 @@ The number of rows modified by the query. -- type: long +format: bytes + The size of the query result. @@ -6567,6 +6569,8 @@ Number of temporary tables created on disk for this query. -- type: long +format: bytes + Size of temporary tables created for this query. -- @@ -6683,6 +6687,8 @@ Number of page read operations. -- type: long +format: bytes + Bytes read during page read operations. diff --git a/filebeat/module/mysql/fields.go b/filebeat/module/mysql/fields.go index c105c1a9666f..ebdc2cbff030 100644 --- a/filebeat/module/mysql/fields.go +++ b/filebeat/module/mysql/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEmN9v3DYMx9/zVxDdywakTvcwDMjDgLXFsALtgKEF9ujyLNrWThZdkc7l+tcPkn0/erYvl+Ru9UOAnC1+P6QkiuJLWNL6Fpq1fHFXAGrV0S28+LD++Pf7F1cAhqQItlXL/hZ+uwIA+MCmcwQlB2gxiPUVaE2QhoDjCkrrSLIrAKk5aF6wL211Cxo6ugIoLTkjt8nUS/DY0E4+Prpu6RaqwF07/DLBEJ8/kiEoAzdzAPHZ19vX1DoQmtya7ZuNtmNf7f04Ix+fTzVBwd5TEd8Ch8EqvHub4hOxvnQU1tlInkLgMJLed/sB7TfsFa2XwcHDOCT7MRqS7Q07DMZDAdmRobMoB29a1HqYvGxudGOrgD3/sADGyo7uyD1S1XGVTY07Ra8hEazosX5OjprT22iJ45Xj6oIzHRUeNdFpReZqG8qEiskolI5RD94cIYRhLygrOoiWd0sflHl5DdaDUMHeyDWgAPYSMXm0bL2C75oFhWx6fXCxvAgvNtx5BS4PmVdolcx2D0cAUIYFAd6hdbhwlEULI6t36DoCK8/yN/BKciF/6NFkejrR1V4vuhqtQyDtgicDi/VUlhrR0D021tN0bjgLkRToHwGEZUmFXhKoYWNLexLRYq109hkT+5XS0tyuykDSOZ1GSF88QSQmjyOefYfTQIqaGpyUXdJ6xeHQ6CluJpuwqin023znN6xQgO6p6JTMdBCKLgTymndC4Xxcb3qrgJ3W5NUWGFNO1LiOf01MOIaUQtx4gEVBItAGe2cdVSQZvEEPxpYlhd2R0OefmLeioZl0iqI5heD5fM68R1HY1R1C5KfFl9a5mV37tCiy2e6SQCjswZbfZHIZNHc8R07FAoua8tpOb+QFsyP0jyP8pyataa8YhCSSyGo7s521aXONx8z5ORCUmpYDxrM5SiSSzYoLJOzu6KGMt+XL2efGyvKS8fJEJtJ9iy3Acf3L8gHCwzx1Yk6GEdhf2/NhRFIEwl3BYGUiHe9zRepnwT2FbYjWAeND0xtPoadBjhg/bg6006I3TZbudhwusD83loFbtY39ms6p7d44TnO5XXCcCtAbsPq0HdIGy8HqOv/SUXeRRLNRgKSwo44TvXFsJrCdc3ksCS+BFY0PqS9KJK4hgpuK/6G6KPH9y/ZyfNH4cTL40TNYb+ieBFaxstkGNw6Wn6bRGwoV5S2KXCD7JOPQG99WVXO9EPjmjlfl0bU8oFIeQc5YAa7blHg2N2WIGuBsY/U67p64BhYEn4VELPvPmwpi99nIpBXAtnXxetBSrHfSyGvgAJ+Tl8mI1eErSV+d7P6U5jP8/zjlNw61Ipfw86tX0BB6Aa1RgT3F6ytC3XkTyCRuO1osALx1XAADwYLi/dZxVc1WWz9ArdrK7c3NarXKWgoFe8wKbm4MFzfD/y+Fwh2Fm1+yX2+MxcqzqC3kJgWI7pW8IZPV2rjpO4v3bBaT4TvsupwQvMPuSyCHamOFxPDOe377GshX1h+u16kOzD6lhvvx/QgemOgTeOPzKaAX7JuS797OIljOQ87teGKPZIITCXYZocUqVeYGuKX+8ifZcaR0nb4A1Otot2cxXUi9mMfDrdDqRB/qHHx/pm3qq5g3lHnZ1+NoAJ3bHAMGFfvrnigHrGgeN1CRp9bZ/4E82TuL6kfimaqC70OnDOR1uGUM27gvUmJF1bf6tscX+2Lc6YuP9RvL3ZF5iItMYnmo1hfjo+T5bv7etoHvbZOK6GLT0kyyQ99g8Lh3aOoonkyks/2O+fZP6jrE0U9o/NQs04fevJxwFwrKDDc4KsVOkbSHh8GJgqNxc2JX/wUAAP//ri9Ugg==" + return "eJzEmN+P2zgOx9/nryB6L3fANNN7OBwwDwdcWyy2QLvAogX20WUs2tZGFl2Rnkz61y8kOz8a24kzM9n6oUAnFr8f0hJJ8TWsaHMP9Ua+uRsAteroHl592nz+/eOrGwBDkgfbqGV/D/+7AQD4xKZ1BAUHaDCI9SVoRZCWgOMSCutIFjcAUnHQLGdf2PIeNLR0A1BYckbuk6nX4LGmvXx8dNPQPZSB26b/ywhDfH5JhqAIXE8BxOdQ71BTq0BoMmt2v2y1Hfvy4I8T8vH5UhHk7D3l8Vfg0FuFD+9TfCLWt5bCZjGQpxA4DKQP3T6j/Y69ovXSO3gch2Q/RkMWB8uOg3EuIHsydBbl6JcGteo/3mJqdW3LgB1/vwGGyo4eyF2o6rhcjK2bo1eTCJZ0qZ+jq6b0tlrieO24vOKXjgoXfei0IzO1NS2E8tEoFI5Rj345QQj9WVBWdBAt77c+KPPqFqwHoZy9kVtAAewkYvJo2HoF39ZLCovx/cH56iq8WHPrFbg4Zl6jVTK7MxwBQBmWBPiA1uHS0SJaGFh9QNcSWHmWv4HXkgn5Y49G09NMVzu96Gq0DoG0DZ4MLDdjWWpAQ49YW0/jueFFiCRHfwEQFgXlek2gmo0t7Cyi5Ubpsi9WcKhR+5WXo4r9Tmnb7nZsIGmdjuOlN54gEhPLCa9/QqWQvKIaR2VXtFlzODY6x81kE9YVhS4F7P2GNQrQI+WtkhkPQt6GQF6zVii8HNe7zipgqxV5tTnGdBQ1buO/JiYjQ0ohHkrAPCcRaIJ9sI5KkgW8Qw/GFgWFfbnoclPMadHQRKpF0YxC8PxyznxEUdj3JELkx8VX1rmJE/20KLLZnZJAKOzBFj9keek19zwnKmaOeUVZZccP+ZLZEfrLCP+oSCs6aBQhiSSyyk4cZ62bTGMJenkOBKW64YCxbkeJRLLdcYGE3QOdy4Y7vox9ZqysrhkvT2Qi3Y/YAhz3v6zOEB7nqZnlAwZgv+1qx4AkD4T7ZsLKSDo+5IrUz4J7ClsfrSPGc583VqHZkLMr3oD/87bYzYvsOHW6E3K4wtndWgZu1Nb2e6phu3NzmuZ6J+Q0FaA3YPVpp6cJloPVTfatpfYqSWirAElhTx0/9NaxicC2zmWxlbwGVjTep8Uokbj6CG5vCud6psT3J9vr8UXjp8ngn57BekOPJLCOXc8uuHGx/GscvaZQUtagyBUyUzIOnfFdxzU1Q4Ef7oZlFl3LAiplEeQFu8NNkxLP9oYNUQOcra3extMT98CS4KuQiGX/ddtd7F8bmLQC2DQuXisair1QWnkLHOBr8jIZsdq/Jemt2e6PaT7D/89jfmPfR3IB/37zBmpCL6AVKrCneO1FqFpvApnEbQebBYB3jgtgIFhSvBc7LsvJTuwfUKk2cn93t16vFw2FnD0ucq7vDOd3/f9fC4UHCnf/Wfz3zlgsPYvaXO5SgOhRyRsyi0prN36f8Z7NcjR8x9OaGcE7ntoEcqg2dk8MH7zn92+BfGn98X4dm9wcUmp4HN6d4MyHnsEbny8BvWA3zPzwfhLBchYyboYf9kQmmEmwzwgNlqlrN8ANdRdDWZxGGmstzkKd6ktmQr+NSztW04Y047kcfo1WR+ZbZ/ln8P2ajrEvY15R5lXXy6MBdG5bJgwqdldFUQ5Y0jRuoDxLI7m/A3l0JhfVT8QzdQ0/h04ZyGt/Q+mPedfExI6rGyHuyhv7fDhBjI/1W8vtie8QN5nE9lGtz4el5vlu/r9pAj/aOjXZ+XZUmmT7mUPvcefQWKkeTbSTs5Lp0VGaWMTVTxgaVSzjRXFaTrgNOS0M1zho1eZI2uNiMVNwsG5K7OavAAAA//8Czml+" } diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml index a64d915d420c..60145e644102 100644 --- a/filebeat/module/mysql/slowlog/_meta/fields.yml +++ b/filebeat/module/mysql/slowlog/_meta/fields.yml @@ -26,6 +26,7 @@ The number of rows modified by the query. - name: bytes_sent type: long + format: bytes description: > The size of the query result. - name: query @@ -74,6 +75,7 @@ Number of temporary tables created on disk for this query. - name: tmp_table_sizes type: long + format: bytes description: Size of temporary tables created for this query. - name: filesort @@ -127,6 +129,7 @@ Number of page read operations. - name: io_r_bytes type: long + format: bytes description: > Bytes read during page read operations. - name: io_r_wait.sec From e97eefe84dcc1cbbea6fa372181660ef1a019f5f Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 22 Jan 2019 12:32:36 +0100 Subject: [PATCH 3/3] Remove unneeded procesor from pipeline --- filebeat/module/mysql/slowlog/ingest/pipeline.json | 7 ------- 1 file changed, 7 deletions(-) diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index 4fb3c0e76e23..b51bcad5a2f8 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -56,13 +56,6 @@ "field": "mysql.slowlog.timestamp", "ignore_missing": true } - }, { - "gsub": { - "field": "mysql.slowlog.query", - "pattern": "\n# Time: [0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9](\\.[0-9]+)?$", - "replacement": "", - "ignore_failure": true - } }], "on_failure" : [{ "set" : {