diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 0b94b20a54d..2eac7ed2637 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -88,4 +88,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Add fields validation for histogram subfields. {pull}17759[17759] - Events intended for the Elasticsearch output can now take an `op_type` metadata field of type events.OpType or string to indicate the `op_type` to use for bulk indexing. {pull}12606[12606] - Remove vendor folder from repository. {pull}18655[18655] +- Added SQL helper that can be used from any Metricbeat module {pull}18955[18955] - Update Go version to 1.14.4. {pull}19753[19753] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8b3c0b200b5..e2a17179f89 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -311,6 +311,7 @@ field. You can revert this change by configuring tags for the module and omittin - Fix config example in the perfmon configuration files. {pull}19539[19539] - Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601] - Fix k8s scheduler compatibility issue. {pull}19699[19699] +- Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898 *Packetbeat* @@ -613,6 +614,7 @@ field. You can revert this change by configuring tags for the module and omittin - Add param `aws_partition` to support aws-cn, aws-us-gov regions. {issue}18850[18850] {pull}19423[19423] - Add support for wildcard `*` in dimension value of AWS CloudWatch metrics config. {issue}18050[18050] {pull}19660[19660] - The `elasticsearch/index` metricset now collects metrics for hidden indices as well. {issue}18639[18639] {pull}18703[18703] +- Added `performance` and `query` metricsets to `mysql` module. {pull}18955[18955] - The `elasticsearch-xpack/index` metricset now reports hidden indices as such. {issue}18639[18639] {pull}18706[18706] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 6924591bd3f..977b1a8de71 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -32004,6 +32004,123 @@ type: long -- +[float] +=== performance + +`performance` contains metrics related to the performance of a MySQL instance + + + +[float] +=== events_statements + +Records statement events summarized by schema and digest + + +*`mysql.performance.events_statements.max.timer.wait`*:: ++ +-- +Maximum wait time of the summarized events that are timed + +type: long + +-- + +*`mysql.performance.events_statements.last.seen`*:: ++ +-- +Time at which the digest was most recently seen + +type: date + +-- + +*`mysql.performance.events_statements.quantile.95`*:: ++ +-- +The 95th percentile of the statement latency, in picoseconds + +type: long + +-- + +*`mysql.performance.events_statements.digest`*:: ++ +-- +Performance schema digest + +type: text + +-- + +*`mysql.performance.events_statements.count.star`*:: ++ +-- +Number of summarized events + +type: long + +-- + +*`mysql.performance.events_statements.avg.timer.wait`*:: ++ +-- +Average wait time of the summarized events that are timed + +type: long + +-- + +[float] +=== table_io_waits + +Records table I/O waits by index + + + +*`mysql.performance.table_io_waits.object.schema`*:: ++ +-- +Schema name + +type: keyword + +-- + +*`mysql.performance.table_io_waits.object.name`*:: ++ +-- +Table name + +type: keyword + +-- + +*`mysql.performance.table_io_waits.index.name`*:: ++ +-- +Name of the index that was used when the table I/O wait event was recorded. PRIMARY indicates that table I/O used the primary index. NULL means that table I/O used no index. Inserts are counted against INDEX_NAME = NULL + + +type: keyword + +-- + +*`mysql.performance.table_io_waits.count.fetch`*:: ++ +-- +Number of all fetch operations > 0 + +type: long + +-- + +[float] +=== query + +`query` metricset fetches custom queries from the user to a MySQL instance. + + [float] === status @@ -37101,6 +37218,16 @@ type: object Non-numeric values collected. +type: object + +-- + +*`sql.metrics.boolean.*`*:: ++ +-- +Boolean values collected. + + type: object -- diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc index 549e8e1dafc..21762cbeb66 100644 --- a/metricbeat/docs/modules/mysql.asciidoc +++ b/metricbeat/docs/modules/mysql.asciidoc @@ -86,9 +86,17 @@ The following metricsets are available: * <> +* <> + +* <> + * <> include::mysql/galera_status.asciidoc[] +include::mysql/performance.asciidoc[] + +include::mysql/query.asciidoc[] + include::mysql/status.asciidoc[] diff --git a/metricbeat/docs/modules/mysql/performance.asciidoc b/metricbeat/docs/modules/mysql/performance.asciidoc new file mode 100644 index 00000000000..e0e47239f21 --- /dev/null +++ b/metricbeat/docs/modules/mysql/performance.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mysql-performance]] +=== MySQL performance metricset + +beta[] + +include::../../../module/mysql/performance/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/mysql/performance/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/mysql/query.asciidoc b/metricbeat/docs/modules/mysql/query.asciidoc new file mode 100644 index 00000000000..fd8cdf650f9 --- /dev/null +++ b/metricbeat/docs/modules/mysql/query.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-mysql-query]] +=== MySQL query metricset + +beta[] + +include::../../../module/mysql/query/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/sql.asciidoc b/metricbeat/docs/modules/sql.asciidoc index 61daf5b8a18..69726a1fe7d 100644 --- a/metricbeat/docs/modules/sql.asciidoc +++ b/metricbeat/docs/modules/sql.asciidoc @@ -8,9 +8,361 @@ This file is generated! See scripts/mage/docs_collector.go beta[] -This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query. +The SQL module allows to execute custom queries against an SQL database and store the results to Elasticsearch. +The currently supported databases are the ones already included in Metricbeat, which are: +- PostgreSQL +- MySQL +- Oracle +- Microsoft SQL +- CockroachDB +== Quickstart + +You can setup the module by activating it first running + + metricbeat module enable sql + +Once it is activated, open `modules.d/sql.yml` and fill the required fields. This is an example that captures Innodb related metrics from the result of the query `SHOW GLOBAL STATUS LIKE 'Innodb_system%'` in a MySQL database: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["root:root@tcp(localhost:3306)/ps"] + + driver: "mysql" + sql_query: "SHOW GLOBAL STATUS LIKE 'Innodb_system%'" + sql_response_format: variables +---- + +.SHOW GLOBAL STATUS LIKE 'Innodb_system%' +|==== +|Variable_name|Value + +|Innodb_system_rows_deleted|0 +|Innodb_system_rows_inserted|0 +|Innodb_system_rows_read|5062 +|Innodb_system_rows_updated|315 +|==== + + +Keys in the YAML are defined as follow: + +- `driver`: The drivers currently supported are those which already have a Metricbeat module like `mssql` or `postgres`. +- `sql_query`: Is the single query you want to run +- `sql_response_format`: You have 2 options here: + - `variables`: Expects a table which looks like a key/value result. With 2 columns, left column will be considered a key and the right column the value. This mode generates a single event on each fetch operation. + - `table`: Table mode can contain any number of columns and a single event will be generated for each row. + +Results will be grouped by type in the result event for convenient mapping in Elasticsearch. So `strings` values will be grouped into `sql.strings`, `numeric` into `sql.numeric` and so on and so forth. + +The event generated with the example above looks like this: + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:09:14.407Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "172.18.0.2:3306", + "type": "sql" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 1272810 + }, + "sql": { + "driver": "mysql", + "query": "SHOW GLOBAL STATUS LIKE 'Innodb_system%'", + "metrics": { + "numeric": { + "innodb_system_rows_updated": 315, + "innodb_system_rows_deleted": 0, + "innodb_system_rows_inserted": 0, + "innodb_system_rows_read": 5062 + } + } + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "488431bd-bd3c-4442-ad51-0c50eb555787", + "id": "670ef211-87f0-4f38-8beb-655c377f1629" + } +} +---- + +In this example, we are querying PostgreSQL and generate a "table" result, hence a single event for each row returned + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + + driver: "postgres" + sql_query: "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + sql_response_format: table +---- + +.SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database +|==== +|datid|datname|blks_read|blks_hit|tup_returned|tup_fetched|stats_reset + +|69448|stuff|8652|205976|1484625|53218|2020-06-07 22:50:12 +|13408|postgres|0|0|0|0| +|13407|template0|0|0|0|0| +|==== + +With 3 rows on the table, three events will be generated with the contents of each row. As an example, below you can see the event created for the first row: + +[source,json] +---- +{ + "@timestamp": "2020-06-09T14:47:35.481Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "localhost:5432", + "type": "sql" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "1bffe66d-a1ae-4ed6-985a-fd48548a1971", + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic" + }, + "sql": { + "metrics": { + "numeric": { + "tup_fetched": 53350, + "datid": 69448, + "blks_read": 8652, + "blks_hit": 206501, + "tup_returned": 1.491873e+06 + }, + "string": { + "stats_reset": "2020-06-07T20:50:12.632975Z", + "datname": "stuff" + } + }, + "driver": "postgres", + "query": "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 14076705 + }, + "metricset": { + "name": "query", + "period": 10000 + } +} +---- + + +== More examples + +=== Oracle: + +Get the buffer cache hit ratio: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["oracle://sys:Oradoc_db1@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"] + + driver: "oracle" + sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit Ratio" FROM V$BUFFER_POOL_STATISTICS' + sql_response_format: table +---- + + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:41:02.200Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "sql": { + "metrics": { + "numeric": { + "hit ratio": 0.9742963357937117, + "physical_reads": 17161, + "db_block_gets": 122221, + "consistent_gets": 545427 + }, + "string": { + "name": "DEFAULT" + } + }, + "driver": "oracle", + "query": "SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) \"Hit Ratio\" FROM V$BUFFER_POOL_STATISTICS" + }, + "metricset": { + "period": 10000, + "name": "query" + }, + "service": { + "address": "172.17.0.3:1521", + "type": "sql" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 39233704 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788" + } +} +---- + +=== MSSQL + +Get the buffer cache hit ratio: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["sqlserver://SA:password@localhost"] + + driver: "mssql" + sql_query: 'SELECT * FROM sys.dm_db_log_space_usage' + sql_response_format: table +---- + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:39:14.421Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "sql": { + "driver": "mssql", + "query": "SELECT * FROM sys.dm_db_log_space_usage", + "metrics": { + "numeric": { + "log_space_in_bytes_since_last_backup": 524288, + "database_id": 1, + "total_log_size_in_bytes": 2.08896e+06, + "used_log_space_in_bytes": 954368, + "used_log_space_in_percent": 45.686275482177734 + } + } + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 40750570 + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "172.17.0.2", + "type": "sql" + }, + "agent": { + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "3da88889-036e-47cb-a88b-275037fa2bc9" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + } +} +---- + +=== Two or more queries + +If you want to launch two or more queries, you need to specify them with their full configuration for each query. For example: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + sql_response_format: table + +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT * FROM pg_catalog.pg_tables pt WHERE schemaname ='pg_catalog'" + sql_response_format: table +---- [float] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 95780308f34..5885ac979d4 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -191,7 +191,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> beta[] +.4+| .4+| |<> beta[] +|<> beta[] +|<> beta[] |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> diff --git a/metricbeat/helper/sql/sql.go b/metricbeat/helper/sql/sql.go new file mode 100644 index 00000000000..b417e3dc923 --- /dev/null +++ b/metricbeat/helper/sql/sql.go @@ -0,0 +1,204 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sql + +import ( + "context" + "database/sql" + "fmt" + + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +type DbClient struct { + *sql.DB + logger *logp.Logger +} + +type sqlRow interface { + Scan(dest ...interface{}) error + Next() bool + Columns() ([]string, error) + Err() error +} + +// NewDBClient gets a client ready to query the database +func NewDBClient(driver, uri string, l *logp.Logger) (*DbClient, error) { + dbx, err := sql.Open(switchDriverName(driver), uri) + if err != nil { + return nil, errors.Wrap(err, "opening connection") + } + err = dbx.Ping() + if err != nil { + return nil, errors.Wrap(err, "testing connection") + } + + return &DbClient{DB: dbx, logger: l}, nil +} + +// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. +func (d *DbClient) FetchTableMode(ctx context.Context, q string) ([]common.MapStr, error) { + rows, err := d.QueryContext(ctx, q) + if err != nil { + return nil, err + } + return d.fetchTableMode(rows) +} + +// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. +func (d *DbClient) fetchTableMode(rows sqlRow) ([]common.MapStr, error) { + // Extracted from + // https://stackoverflow.com/questions/23507531/is-golangs-sql-package-incapable-of-ad-hoc-exploratory-queries/23507765#23507765 + cols, err := rows.Columns() + if err != nil { + return nil, errors.Wrap(err, "error getting columns") + } + + for k, v := range cols { + cols[k] = strings.ToLower(v) + } + + vals := make([]interface{}, len(cols)) + for i := 0; i < len(cols); i++ { + vals[i] = new(interface{}) + } + + rr := make([]common.MapStr, 0) + for rows.Next() { + err = rows.Scan(vals...) + if err != nil { + d.logger.Debug(errors.Wrap(err, "error trying to scan rows")) + continue + } + + r := common.MapStr{} + + for i, c := range cols { + value := getValue(vals[i].(*interface{})) + r.Put(c, value) + } + + rr = append(rr, r) + } + + if err = rows.Err(); err != nil { + d.logger.Debug(errors.Wrap(err, "error trying to read rows")) + } + + return rr, nil +} + +// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. +func (d *DbClient) FetchVariableMode(ctx context.Context, q string) (common.MapStr, error) { + rows, err := d.QueryContext(ctx, q) + if err != nil { + return nil, err + } + return d.fetchVariableMode(rows) +} + +// fetchVariableMode scan the rows and publishes the event for querys that return the response in a key/value format. +func (d *DbClient) fetchVariableMode(rows sqlRow) (common.MapStr, error) { + data := common.MapStr{} + + for rows.Next() { + var key string + var val interface{} + err := rows.Scan(&key, &val) + if err != nil { + d.logger.Debug(errors.Wrap(err, "error trying to scan rows")) + continue + } + + key = strings.ToLower(key) + data[key] = val + } + + if err := rows.Err(); err != nil { + d.logger.Debug(errors.Wrap(err, "error trying to read rows")) + } + + r := common.MapStr{} + + for key, value := range data { + value := getValue(&value) + r.Put(key, value) + } + + return r, nil +} + +// ReplaceUnderscores takes the root keys of a common.Mapstr and rewrites them replacing underscores with dots. Check tests +// to see an example. +func ReplaceUnderscores(ms common.MapStr) common.MapStr { + dotMap := common.MapStr{} + for k, v := range ms { + dotMap.Put(strings.Replace(k, "_", ".", -1), v) + } + + return dotMap +} + +func getValue(pval *interface{}) interface{} { + switch v := (*pval).(type) { + case nil, bool: + return v + case []byte: + s := string(v) + num, err := strconv.ParseFloat(s, 64) + if err == nil { + return num + } + return s + case time.Time: + return v.Format(time.RFC3339Nano) + case []interface{}: + return v + default: + s := fmt.Sprint(v) + num, err := strconv.ParseFloat(s, 64) + if err == nil { + return num + } + return s + } +} + +// switchDriverName switches between driver name and a pretty name for a driver. For example, 'oracle' driver is called +// 'godror' so this detail implementation must be hidden to the user, that should only choose and see 'oracle' as driver +func switchDriverName(d string) string { + switch d { + case "oracle": + return "godror" + case "cockroachdb": + return "postgres" + case "cockroach": + return "postgres" + case "postgresql": + return "postgres" + } + + return d +} diff --git a/metricbeat/helper/sql/sql_test.go b/metricbeat/helper/sql/sql_test.go new file mode 100644 index 00000000000..1045c8d5b87 --- /dev/null +++ b/metricbeat/helper/sql/sql_test.go @@ -0,0 +1,188 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sql + +import ( + "math" + "testing" + "time" + + "github.com/elastic/beats/v7/libbeat/common" +) + +type kv struct { + k string + v interface{} +} + +type mockVariableMode struct { + index int + results []kv +} + +func (m *mockVariableMode) Scan(dest ...interface{}) error { + d1 := dest[0].(*string) + *d1 = m.results[m.index].k + + d2 := dest[1].(*interface{}) + *d2 = m.results[m.index].v + + m.index++ + + return nil +} + +func (m *mockVariableMode) Next() bool { + return m.index < len(m.results) +} + +func (m mockVariableMode) Columns() ([]string, error) { + return []string{"key", "value"}, nil +} + +func (m mockVariableMode) Err() error { + return nil +} + +type mockTableMode struct { + results []kv + totalResults int +} + +func (m *mockTableMode) Scan(dest ...interface{}) error { + for i, d := range dest { + d1 := d.(*interface{}) + *d1 = m.results[i].v + } + + m.totalResults++ + + return nil +} + +func (m *mockTableMode) Next() bool { + return m.totalResults < len(m.results) +} + +func (m *mockTableMode) Columns() ([]string, error) { + return []string{"hello", "integer", "signed_integer", "unsigned_integer", "float64", "float32", "null", "boolean", "array", "byte_array", "time"}, nil +} + +func (m mockTableMode) Err() error { + return nil +} + +var results = []kv{ + {k: "hello", v: "world"}, + {k: "integer", v: int(10)}, + {k: "signed_integer", v: int(-10)}, + {k: "unsigned_integer", v: uint(100)}, + {k: "float64", v: float64(-13.2)}, + {k: "float32", v: float32(13.2)}, + {k: "null", v: nil}, + {k: "boolean", v: true}, + {k: "array", v: []interface{}{0, 1, 2}}, + {k: "byte_array", v: []byte("byte_array")}, + {k: "time", v: time.Now()}, +} + +func TestFetchVariableMode(t *testing.T) { + db := DbClient{} + + ms, err := db.fetchVariableMode(&mockVariableMode{results: results}) + if err != nil { + t.Fatal(err) + } + + for _, res := range results { + checkValue(t, res, ms) + } +} + +func TestFetchTableMode(t *testing.T) { + db := DbClient{} + + mss, err := db.fetchTableMode(&mockTableMode{results: results}) + if err != nil { + t.Fatal(err) + } + + for _, ms := range mss { + for _, res := range results { + checkValue(t, res, ms) + } + } +} + +func checkValue(t *testing.T, res kv, ms common.MapStr) { + switch v := res.v.(type) { + case string, bool: + if ms[res.k] != v { + t.Fail() + } + case nil: + if ms[res.k] != nil { + t.Fail() + } + case int: + if ms[res.k] != float64(v) { + t.Fail() + } + case uint: + if ms[res.k] != float64(v) { + t.Fail() + } + case float32: + if math.Abs(float64(ms[res.k].(float64)-float64(v))) > 1 { + t.Fail() + } + case float64: + if ms[res.k] != v { + t.Fail() + } + case []interface{}: + for i, val := range v { + if ms[res.k].([]interface{})[i] != val { + t.Fail() + } + } + case []byte: + ar := ms[res.k].(string) + if ar != string(v) { + t.Fail() + } + case time.Time: + ar := ms[res.k].(string) + if v.Format(time.RFC3339Nano) != ar { + t.Fail() + } + default: + if ms[res.k] != res.v { + t.Fail() + } + } +} + +func TestToDotKeys(t *testing.T) { + ms := common.MapStr{"key_value": "value"} + ms = ReplaceUnderscores(ms) + + if ms["key"].(common.MapStr)["value"] != "value" { + t.Fail() + } +} diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index f5935a8141c..39fadeea0e0 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -112,6 +112,7 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/munin/node" _ "github.com/elastic/beats/v7/metricbeat/module/mysql" _ "github.com/elastic/beats/v7/metricbeat/module/mysql/galera_status" + _ "github.com/elastic/beats/v7/metricbeat/module/mysql/query" _ "github.com/elastic/beats/v7/metricbeat/module/mysql/status" _ "github.com/elastic/beats/v7/metricbeat/module/nats" _ "github.com/elastic/beats/v7/metricbeat/module/nats/connections" diff --git a/metricbeat/module/mysql/fields.go b/metricbeat/module/mysql/fields.go index 283f3e4df4b..8b04d180f45 100644 --- a/metricbeat/module/mysql/fields.go +++ b/metricbeat/module/mysql/fields.go @@ -32,5 +32,5 @@ func init() { // AssetMysql returns asset data. // This is the base64 encoded gzipped contents of module/mysql. func AssetMysql() string { - return "eJzkXF1z2zazvs+v2MlN0xnb0+tcnBnXcVrPxElqOe2cK3ZFrkgcgwADgJLZX38GC1KiJFKiZFHO21c3iSWSeHaxePYDC17CE1XvIa/sd/kGwAkn6T28va8mf3x6+wYgIRsbUTih1Xv4nzcAAPwbWDJzMmAdutJCTs6I2EKspaTYUQIzo/Nw6dUbAJtp46JYq5lI38MMpaU3AIYkoaX3kOIbgJkgmdj3PMYlKMxphct/XFX4S40ui/qbDnD+8zff9TfEWjkUyoLLaInQZehgQYZAT/2va1CXj/hekqmu6j/bwNrgUpRkMAoqWP7aBdR/lsJOyWHr+x4hWJC1EYYLNK34inpu/GyxPKAV/MZPvGoNsyldW0IsClmt/dIn3R5J/OfaP6wBFUa92rioC0sbj9aatn5sICW6nMqun/fg8p/f9QL0zJFikUUwbOPteGGEo0tLLihDqBR06S717FKbhAy8K9CglCTFP+hHAJrNRCxIxdXPK/F2SSRHlmglwQItWA1W6gU4HQSq7Wd1jXAZZCLNvA7ou9I/2WBdQTEJEBqvoM2pC58/UZZkIZbakvFj/AKGZuG/CKkhdGQgxcKvggWRCmBQJTBD28JhB+huIVSiF6No73pOBlOCRFiHKqYlXNaMdYxY6oX/b6xVXBpDyslqqSVWXY8MDf6YjDvV4roh48RMxMEGX7TIEips1Aj+6tplRcLc21Uw1RgVTAkKba2YtjQuFDRLEd4V2pFyAiUklBoi0DPYWKhDVqdQCT1HVvzTrwepVXqcFh4zAlXmUzIeHSlnBFkvhufueG0+GccgvI7MHMdhlGbWVpidQWUx9jdZMBSTmHvGzIQkwPavYKiQXhjqW9fLNSFL68icbFmEx71sQfiwJRLJGCagHcqWQmvpISf/jc1EAXGGKiULGRYFKUoGWMFI9noTSK4Ft4a5tNmAfgjCzZBpHeMTVQttuhQ+AOYkTLU3z0zYpUpjnRdakXJX8OhpRNgLWGTkvJ/z4JVOCIT1LOH8zQhfH+7urx/+F7SBz18+R82fqwftsWSd5+J0/M5P+/Gjp7VV7+ONoAYfSemSFcth08u9+/F2fLRvX4kyyLtrpTgV6jSBbhPfg/5uxqYanKGw8OXjx4uV8WZoQWkHFbnV4Bx4qSosB9peDVtG5P2SsJBj5b1s4r2uhlzYkLuVhh3SFdxkFD/xI8kYbUDqFGbaQGF0QQYSganS1ol4H+HTfJMHjl4jcDu38PGopUFzEW+u1X2TNQQRAHwS1oWM7du3uw8/MTOhlDxnNgzc5KCdJLouYLg63Buj8vNt6P/0OgNDqZyQUOkSDHEi438VJqTTiZ+kmKztdcawQdX9rPEyps70ImiG4xaFMozWGOvtnxP4arTTsZZ7rGgm9SKK3Wbgc7QpffRZyY1Wzmh5JNsWWNqtxQ8n4VsfOc5MTbJeWSInsMJzmdeb9JnUx0/fJr/D5PH68duEmcuzGgfQTSzWMHQA2ix1r0kuNJi20tufOwWa3aafeXsBmV5AXsZZqDlInHsEqecnn9v5hDnRi0MjhAAqUv1BwssCb8eRV1Bc4aMa4Z1XrYpghTmhLU3ILBQqbSnWKhmyZgzF8xFwP5ArTV39WQVhH2+ir9ffJrdAc8/n6/6gCcovQKhYlomfDZdpS+uX2bVwpv35pqR4Isi1XQYfczQCp5Js8D2xLv3qZfbniEsrgkRTcEaGLDkPzVRB20xKZTCHtSrbLg4i1U/OZ9XnD6ioRkl+1UfLEKWTCDtUtUdNfqlY+l6S55agowsfEHMAdNEQNRPOKjpqhYD7MOt4K1k9mrM/+0myBcU+bz5N5jedRTjVxo3CQhu5H+tiPaVelXYZRajsBoZduy5k3EIBPVNc7tA7bFSfohkKWRp6Tfk8BEo2Ch6ObF9+BVs51iuhXxr8Sz1Cl60PhLlu836UHkuHHdbeBvq9pLIrKoF9Oh0IGFqFhHdC+QzMoSJd2p9Bkkpd1pAKC8Nwduh3C3qE8z50e+KuAwR4WEJrMGPIKRPQczLLMtyQmKzbm0Crlq6VFQkZnMoKJJqUCxao4JerX3yMosIyWrqpOisIxf1VPR3Qcom9dzhkV1cBGlrV8nzQuBBSQkqKjI+KEKTmPL4dRrrMaOekUOlBc5Xj88im5v1Xjs8iL/Ne8zpsloaIJdQ5xBJqTLFWzFVIrM5BsZv1Yqxsk5SgrfKw3+mJ+AlSg6qUaIQbGD7252EnI18fGv5ryNer7Acl38kSWjf5Dk2GX0a8QiW8n9FDgz5GVuQW2jzxt2WaFaUDYe2BGn1NilwZwb+KIk8m1hlKZHdNYSz0bSyzmo+T+7pKEfhzT55lCJPuPo6j6tB/tfZN6j4gYcMgzNZxTIXjpFXs3fBrQo1TZYF/ccQzIdc8+aV7f+UotYct79eK1JbRFxeFQ62NK74DbHFauRGzOSv+oReCbccVp5rzhxYHv6z1AR1G51GhH2q1LT6EZp7oLGGYH+ZwYGfS2uHg2CLPhC5Y/1S4AzGORTI7OabBB+84cnW6vXiHNMe8AtVs67S3o6CbTHY1W56oy7LNOesNrs1nZ8dlKPSdihmv67rhy3ygFKTGqbKtVd7rTeu+0ifFWNYbFwERJIISbljUpeOew7C9Qa0n1RvSckieWN81vqB1yROdo7xw1q+9emz/Xy9Au8d6jyOdCiX1JqwjDWavJWCc0VUi7FNU2lP1GO0ZbaSBdrHY0cvtV/8wXmwHL7KeOJhv1SZHNyLhrhknj7IR1qGUDQuMtF23T8YjxWhc234BGvAu83nMySziMTzuOJvgBTBK3+E67/IwjeRDiJL7uM8ALIxzCLKepqsRsK06rAajM6VSonP402Krx9mDbENn3u12Wv3wXePux3cay0gOyuVFcE+ON97H9lB+uJk400DjibTq/JdYnWuquG9wdMUJZcm4qJvVTz4aJyujTtFMljaL6trkKOs1x+eIe69G5gVd0GYNeCRLO8sKtc4Q5uMTwegkoPmYQTeBnsYhBOM9z9wnJGlHcf5UHvfD7afbx9um5F3vK3DnbVkMOrdjt8+CnR7l3efJ7cPj0SgtSdrRJX0qlJPbT7c3x6Msi2TXdsypUH77+uH6wBlvbYH5e164trph8aZQzo2LoRmsVagK1YTQ39/0JtZ9oXXWunnnQrhMKLBOG+K28dRgbi+gDM2O/ql/lGRDyaZ55BXcuVVfI9c24ebLffT17vNvoA3/f/J4/Xg3eby7We6z7YtSvzfjvKraltrSqj7LXN/VJJqtLbFp1aSc3EbjlXG8jtnIVhpen8MeZV9sqLr5+/4x+vpw+/X64bb1zc2nL5Pbi9X83D9GD7eT28eh85OhSuSpDu7t35XrOFu10xoGWMS2VSyPJdx8ub+/e2xN3wAeOpPncSKnulxq9MJChnOCKZGqATRHS9idD4BNz0HmSOr4aST0zZ6xik29GtyaPc+0AcI4gxil9KtK8IJpAXv3M8xKxcHpBSwyEWd1h5qUFeg4Lo2FuiluSqkI6bBfhaQSPoATx2RtOJXuah4Mp8EGqCg3JhLqDPbXzO1SY6UlC8hcgSkBqVQo+smCXii4L6UTlw+oUoIHwgREXkhWb9gY5bZzFjUIP+QQhqECzRg2fL3RDU/NWFBkaMMJoIW+DH+E1e4D03D6bNgRDByh52tIE9dMmN4uy1M2t3SZCY/NR8krPtBSnxznYz5eIwN7dJ5os1lkXAEMsXfn9ehhAnougylaSkArQI9oIHbZ3+N6JvDLdqInWpuFKxgmgaLnH0ACj4JnQSgWZN+xXVjnja5u9zNL4FEIXdpjpTCdXaNnFGJrFczEsw8RtRXhEO5gOaIfz6b839wDMxM+oB3A6FrKKY4Wk3RA965p09l6eQoyM21ynp+AaeWbhqTROKdCi3E6Pg6URGJM/pcG0iHwox9nRjwS4IngiO4Qcc5ULmivhzBkvbz53CnHZUPeiWDEmbGGytRQrKt6ltLJdFOOkyaDy/37cjYjExVd79HaFe3t0Up/pLfM78q82H732+bgvbw6iFUf2Y3p1PhkxactasU0gVljbWpvh/yWGJJJw653SukPv9YKAq+gC3BGpCmZdnXDOe5Xn9WTFrUUGrGM6CKblY4PcWvTe5nSi52OdXXmE5PX1tsCTQ5l0aMmrxvvs7xe2OQtufACqZSPBRhDttAqHKbW/uH1O9GASYf1L3I6WNusGa/tUEXsUTZfNVTZfe0U+zKhQXrelQ2t99T2Bg17g4/B4Uf7WP9mF0jvgmh6/fglBehwX0jVkkoY15ckjSVWvPUCpiBes+gZU22lvSIPMht+xn+J2ay5vj3K27KXH9Nctg3l5aYBm5vQnT0248jTHZowilqm5etzXiKVob6jaWOK5Ic92cxIdD0tY2OLUY98MklyYeNXEGMfAXChHQ3BtLRVuye5alXeUUodeus5YUlyoYR1PvyYEx9SzAiTC7BlnAGG+oTU8ZOFuhaKCRZ8bYY22/vyx3W9sas4s2NqevX7ok5+NWe6c/thb+X2jK4Hsz4IcAbb42qJUPVeYnd0Gk56Y3LJUDnz9XpRTTPiYGvh+6P6JWT/0SK3TgzYchreWeNktXzBWnNMIMO5995hnfJGSAjPd59W7NNcf41wTK29NagSnb9tKcRzlnAC643roMPBwjRu9TX8hk5FjDLYQINjWJDa/ersMxJFX09hG8bYOqv3futFE+tS1q0G6ISdVavIaI2RUSWQYbKszibCUOyXC1+eCPt0gO14+t+V0Y+hDLt8y+t2TYTdUQ+TJJXC3CtPVhv5eZObh/cRbqXd/NACDebkyLSfM1hTCxQuOluU+VmbPIgZulKhn17Du4WbqGdFrVfwV0aquUMRJat6vjZ1Zze/Ni+luscFYklYu3uOk3COQuJU0kXznJA9WLA6p7XEJOxe8itnUNTVXjbutfmd+WAqq1tgml1k/tcuN/Hr1/+SpfCoQXzCaop2UuGpalMbRzItJDoUsvuC5v8PAAD//zEqUrk=" + return "eJzkXc1z2ziyv+ev6JrLZKocvbnsYVP1tsrrOLuuip2s7ey8d9K0yBaJNQgwAChF89e/QoNfkkiJ+qCct6tLbIsEft3obzSQd/BCq/eQrew3+QbACSfpPfx0v3r6x6ef3gDEZCMjcie0eg9/eQMAwN+BJbMgA9ahKyxk5IyILERaSoocxTA3OguPTt4A2FQbN420movkPcxRWnoDYEgSWnoPCb4BmAuSsX3Pc7wDhRk1uPzHrXL/qNFFXv6lA5z//M5v/Q6RVg6FsuBSqhG6FB0syRDomf92DWo9xLeCzGpS/toG1gaXoCSD08CC+tsuoP5TEzsjh62/9xDBhKzNMJyg2YqfKNfGrxbTA1rB33jESWuaTeraFGKey9XaN33U7aHEf679YBWoMOtk46EuLG08Wmva+rKCFOtiJru+3oPLf/6ul6DnjhSTLIJgGy/HSyMcvbPkAjOESkAX7p2ev9MmJgNvczQoJUnxB/oZgOZzEQlS0eqXhrxdFMmRKWooWKIFq8FKvQSnA0Gl/DTPCJdCKpLU84C+Kf2zDdIVGBMDofEM2ly68PknyoIsRFJbMn6OX8HQPPyIkBhCRwYSzL0WLIlUAIMqhjnaFg47gHdLoWK9HIV71wsymBDEwjpUEdVwmTPWMWKpl/7HSKuoMIaUk6uaS8y6Hhoq/BEZdy7luiHjxFxEQQZPUrKYcjutCH917jIjYeHlKohqhApmBLm2VsxaHBcKKlWEt7l2pJxACTElhgj0HDYUdYh2ChXT96kVf/TzQWqVHMeF55RAFdmMjEdHyhlB1pPhbXe0tp6MYxBeR2aB41iUatUazM6gshj5lywYikgsvMVMhSTA9rdgKJeeGOrT61onZGEdmbOpRRjuNIXwYctUxGOIgHYoWwwtqYeM/F9sKnKIUlQJWUgxz0lRPEAKRpLXm2DkWnBLmLXMBvRDEG6GTOsYX2i11KaL4QNgPoWl9uKZCluzNNJZrhUpN4Fnb0aEvYJlSs77OQ9e6ZhAWG8lnH8Z4cvj3f314/+CNvDw+WFa/doMtEeSdZaJ89l3Hu3Hj57WtN7HG4ENPpLSBTOWw6bTvfvxcny0b29IGeTdtVKcCnWKQLeI70F/N2dRDc5QWPj88eNVI7wpWlDawYpcMzkHXmoV1IG2tWFLiLxfEhYyXHkvG3uvqyETNuRuhWGHNIGblKIXHpKM0QakTmCuDeRG52QgFpgobZ2I9hl8WmzagaN1BG4XFj4epRq0ENGmru5brCGIAOCTsC5kbF+/3n34mS0TSslrZsPEVQ7aaUTXCQxPh3cjVH69Df1Lr1tgKJQTEla6AEOcyPhvhQnpdOwXKSJre50xbJjqfqtxmqVO9TJwhuMWhTLMVgnr7T+f4IvRTkda7pGiudTLaeQ2A5+jRemjz0putHJGyyOtbY6F3VJ+OIu99ZHj3JRG1jNLZARWeFvm+SZ9JvXx09env8PT8/Xz1ye2XN6qcQBdxWKVhQ5AK1X3nORCg2kzvf25U6DZbfqVt1eQ6iVkRZSGmoPEhUeQePvkczufMMd6eWiEEEBNVX+QcFrg7TjyCozLfVQjvPMqWRGkMCO0hQmZhUKlLUVaxUN0xlC0GAH3I7nClNWfJgj7eDP9cv316RZo4e35uj+ogvIrECqSRexXw6Xa0vpjdi2caX++KileCDJt6+BjgUbgTJINvifShddetv4ccWlFEGsKzsiQJeehmVXgNhulIojDWpVtlw0i1W+cL8rPH5BRFZO81k/rEKXTEHawag+bvKpY+laQty2BR1c+IOYA6Koy1GxwmuioFQLuw6yjrWT1aJv94BfJ5hT5vPk8md9sPsWZNm4UK7SR+zEv1lPqprTLKEJlN1jYtedCxi0U0HeKih18h43q03SOQhaGXpM+D4HijYKHI9uXX8FWjvVK6GuBP9UjdMn6QJjrMu9n6ZF02CHtbaDfCiq6ohLYx9OBgKFVSHgrlM/AHCrShf0FJKnEpZVRYWIYzg7+bkGf4qIP3Z646wACHmtoFWYMOWUMekGmLsMNicm6vQm0aulaWRGTwZlcgUSTcMECFfw6+dXHKCqoUe2myqwgFPebejqg5RJ773TIrm4FaKip5fmgcSmkhIQUGR8VIUjNeXw7jHSp0c5JoZKD1irD7yOLmvdfGX4XWZH1itdhqzSELKEuQZZQY5LVWK5c4uoSJnazXowrWyUlaFdZ2O/0hvgFEoOqkGiEGxg+9udhZzO+PjT8tzG+nmU/qPF9qqF1G9+hyfBphleomPczesygj5EVuaU2L/zXIknzwoGw9kCOvqaJbITg38pEno2sC5TI7qrCWOjbqLOaj0/3ZZUi2M89eZYhjLv7OI6qQ//W2jcp+4CEDZOwtY4iyh0nrWLvhl8VapwrC/yNI54nctXIp+79FaPUHra8XytSq6MvLgqHWhtXfAfI4mzlRszmrPiDTgTbjivOteaPLRt8WusDOpxehoV+qmZbfIiZeaGLhGF+msOBXYhrh4NjibwQuiD9M+EOxDiWkdlpYyp88JYjV6fbyjukOeYVTM02T+v9AjJzbbKNlqVui7Kr47I1Tqvfsuq1NCSx3FT1vq/1cGgZCK2xIbJeA9LX/Lmr+TLUn7n5kzL/4y6j+EiRNrGF+umqfG2LLEMj/iiLh1FKGXJaH4uEtsoB/SZ4n+n8OcPvEx97m8kShfv5JMG4L+NRP1II6KvYraGmLs+j45qFf2w7yqrh+QBvYolUP7K4O4xbQ/bswaCDZSrKHa/AR95J4+K/d4jcLuAn68fzrUDlhKTJn/90Gq98oPvnP7nUS6OfWciGW7UweLFV0eqKu+REVO1l9TvCLuFocDn63vXlGq4vLeUoxa5n1JonbAcn1qE5jSUPtcXbkpf+yXGRnE1+qxaT0+S3guZwJmkq9NQPd2S3RGUfeCy4+6/PjM16m8DNhQdqu579i3Y0TPQVVYYUQYKs7MwO+1OqLbqfguT5oXfO2vvAEXM+M487R1xrLJ30THqGvPEBG5njucq9JLTAO9zLlMLO4ro8BKHkxwwLDMWTug+uKnmUMlu/2QmAZ2EfaUSGppSyCTx8/fSJq9abo4Q3lK4evFOWjLOsGmHnNAZMvDd2cPfw4fZ/pg/X97fw3zzi7shqMicXpWeyKCgl8HigcwrdUBb+Ar9uRSS8Z7s3FjnmVAiP/HsVlJALeMhCVFinsyrxbnqKClt14K8HKLuOgfS2ah4eU53p+Mqki23J0Giq3EE9V8p5XW7InlZckKLTJ527t7zsBuzbU6YIi7IjJCCCWHgbIVyqC8eHOULfCLVGKjv95JACfPnW+ISWe8noHGW5d666mrsK2duH1/ZUKGZCSb0J60iB2SsJGKU0iYV9mRb2XM3be2YbaaJd6eHR6vZXPxgr28FK1lNg5Fd9fOpGzGTXhJNn2aiXeW9SWoGR+qD20XgkGVXNYD8BdRSbGsKtlONoiXgOwx0nE6wAoxzoWLe7PE1F+RBDyQfkLgAszHMIsp5u9hGwNa3rg9GZQinROf15sZXz7EG2wTPvdjulfng7XvfwncIykoNyWR7cEwfs5zKWO6ebiwtNNB5JzZFKiatLLRUfyBidcYLzs2m3VT/7bFwFHnWJ5rKw6bTc9B1FXzP8PuWm9pHtgs636o4jSdpFNNQ6Q5iNbwhGNwKaz292G9DzOIQgvJdZ+5gk7eh6OJfH/XD76fb5ti5phIYNPtJU5IMORNvtQ/bnR3n38HT7+Hw0SktyVzX1XCifbj/d3hyPssh7NkjOi/Lrlw/XB654u+QmTtatbljNNlzZZd8qVIVqQjg4WR36KA/ckO18cylcKhRYpw3xebzEYGavoAinSPyo/yjIhpJNNeQE7lxzYIRLm3Dz+X765e7hb6AN//z0fP189/R8d1M3MO2LUr9V87wq22puaVVeElO+VSWarV6j2apKOXkj0zPjeB6zkDUcXl/DHmZfbbC6+v3+efrl8fbL9eNt6y83nz4/3V4163P/PH28fbp9Hro+KapYnutGhP3tTh2H1ndKwwCJ2JaK+rznzef7+7vn1vINsEMX8jxOZNVWh9FLCykuCGZEqgRQndlldz4ANn0PNE+ljl5GQl8146nIlNrg1uR5rg0QRilEKKXXKsEK0wL29heYF4qD06tykzu0/ku5Ah1FhbFQnjaYUSJCOuy1kFTM2yNRRNaGzQZX2sHNzYZeFmXGTIW6gPxVa1tzrLBkAdlWYEJAKhGKfraglwruC+nEu0dUCcEjYQwiyyWzN3Sc8Xk+JjUQP+R0q6EczRgyfL1xzJCquSBP0YY9waV+F34J2t7ayBp0HgBHaKYfsjE8F6b3+Mo5u4a7xITn5jt6VnxSuLySp9wpxaHNzy+02YU7LgGG2LuzPnqYgN6WwQwtxaAVoEc0ELvsPzx0IfB1n/YLra3CBIZRoLq7Vi5MgUfBqyAUE7LvPhRYtxtdxwgvTIFHIXRhj6XCdB7HuSARW1owF999iKitCLebDKZj+uPJFPekoUOuzuwipSZDSznD0WKSDujeNW06W09P2U/J6xMwNb5pSBqNC8q1GKeV9kBKJEbkv6kgHQJ/+uOsiEcCvBAc0R1CzoXKBW19CFOW6s0XenBcNuSyKSMujDVUpoZibepZSsezTTrOmgzW+/fFfE5mmnddULor2tvDlf5Ir87viizfvlR3c/JeuzrIqj6zG9OJ8cmKT1tUY2mCZY20Kb0d8vV7JOPKut4ppT/8tWQQeAZdgTMiSci0qxvO8UHAeblo0xZDp0wjuqlNC8e342jT+5jSy52OtblMA+PX5tsSTQZF3sMmzxvvszxfWOQtuXAzZ8LnLY0hm2sVbqnRfvDysllgo8P8FxkdzG3mjOd2qCL2MJufGsrsvnaKfZnQID7vyobWDyv1Bg17g4/B4Uf7vqTNLpBehah6/fj2J3S4L6Rqd54b15ckjUVWtHWzZSCvUnrGVEppL8mDxIbH+A8RmzXXt4d5W/LyY4rLtqCcLhqwuQnd2WMzDj3doQmjKGmqe4hPocpQX2/9mCT5ac+2MhJdT8vY2GSUM5+NkkzY6BXI2GcA6nMws8Ku2j3Jq1blHaXU4dAiJyxxJpSwzocfC+LbH1LC+ApsEaWAoT4hdfRioayFYow5P5uiTffeqr3ON3YVF3ZM1SHIvqiT7zxPdm4/7K3cXtD1YNoHAS4ge1wtEarcS+yOTsMVOhi/Y6ic+Xq+qKoZcbC08PvT8nbX/9ckt04M2GIWLgN0clXfXFsdE0hx4b130FPeCAnh+e5rIPo4118jHJNrPxlUsc5+ajHE2yzhBJYb14GHg4mp3Opr+A2diAhlkIEKx7Agtfv/JLmgoejrKWzDGJtn5d5vqTSRLmTZaoBO2PmqiYzWLDKqGFKM6+psLAxFXl348VjYlwNkx5v/XRn9GMyw9fX52zURdkc9liReKcw88+RqIz+vcvNwFnEr7eZBczSYkSPTHmcwp5Yo3PRiUeaDNlkgM3SlQr95Df9pQxX1NKZ1Ar+lpKo3FFHc1PO1KTu7+T7ihMoeF4gkYenuOU7CBQqJM0lX1Tghe7BgdUZriUnYveS7/PjQb9gaRre+vnMfTKVlC0y1i8z/2noTv/x/FciGs83D7AmzabrTFJ6rNrVx14WFWIdCdl/Q/H8BAAD//468214=" } diff --git a/metricbeat/module/mysql/module.yml b/metricbeat/module/mysql/module.yml index ea9854af761..d07603c6179 100644 --- a/metricbeat/module/mysql/module.yml +++ b/metricbeat/module/mysql/module.yml @@ -1,3 +1,6 @@ dashboards: - id: 66881e90-0006-11e7-bf7f-c9acc3d3e306 file: Metricbeat-mysql-overview.json +name: mysql +metricsets: + - performance diff --git a/metricbeat/module/mysql/performance/_meta/data.json b/metricbeat/module/mysql/performance/_meta/data.json new file mode 100644 index 00000000000..57d4a044d93 --- /dev/null +++ b/metricbeat/module/mysql/performance/_meta/data.json @@ -0,0 +1,62 @@ +{ + "@timestamp": "2020-07-13T13:43:28.495Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "event": { + "duration": 1379935, + "dataset": "mysql.performance", + "module": "mysql" + }, + "metricset": { + "name": "performance", + "period": 10000 + }, + "service": { + "address": "tcp(172.17.0.2:3306)/", + "type": "mysql" + }, + "mysql": { + "performance": { + "events_statements": { + "digest": { + "text": "SELECT @@SESSION . `auto_increment_increment` AS `auto_increment_increment` , @@`character_set_client` AS `character_set_client` , @@`character_set_connection` AS `character_set_connection` , @@`character_set_results` AS `character_set_results` , @@`character_set_server` AS `character_set_server` , @@`collation_server` AS `collation_server` , @@`collation_connection` AS `collation_connection` , @@`init_connect` AS `init_connect` , @@`interactive_timeout` AS `interactive_timeout` , @@`license` AS `license` , @@`lower_case_table_names` AS `lower_case_table_names` , @@`max_allowed_packet` AS `max_allowed_packet` , @@`net_write_timeout` AS `net_write_timeout` , @@`performance_schema` AS `performance_schema` , @@`sql_mode` AS `sql_mode` , @@`system_time_zone` AS `system_time_zone` , @@`time_zone` AS `time_zone` , @@`transaction_isolation` AS `transaction_isolation` , @@`wait_timeout` AS `wait_timeout`" + }, + "count": { + "star": 2 + }, + "avg": { + "timer": { + "wait": 1.78294e+08 + } + }, + "max": { + "timer": { + "wait": 1.89622e+08 + } + }, + "last": { + "seen": "2020-07-13 10:04:47.709230" + }, + "quantile": { + "95": 1.90546071e+08 + } + } + } + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "mcastro" + }, + "agent": { + "id": "803dfdba-e638-4590-a2de-80cb1cebe78d", + "name": "mcastro", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "f87e6edc-2f37-45f2-9644-b67b1834abfd" + } +} diff --git a/metricbeat/module/mysql/performance/_meta/docs.asciidoc b/metricbeat/module/mysql/performance/_meta/docs.asciidoc new file mode 100644 index 00000000000..2507cb0a06b --- /dev/null +++ b/metricbeat/module/mysql/performance/_meta/docs.asciidoc @@ -0,0 +1 @@ +`performance` metricset fetches performance related metrics (events statements and table io waits) from MySQL diff --git a/metricbeat/module/mysql/performance/_meta/fields.yml b/metricbeat/module/mysql/performance/_meta/fields.yml new file mode 100644 index 00000000000..9e2c0940b73 --- /dev/null +++ b/metricbeat/module/mysql/performance/_meta/fields.yml @@ -0,0 +1,49 @@ +- name: performance + type: group + description: > + `performance` contains metrics related to the performance of a MySQL instance + release: beta + fields: + - name: events_statements + description: Records statement events summarized by schema and digest + type: group + fields: + - name: 'max.timer.wait' + type: long + description: Maximum wait time of the summarized events that are timed + - name: 'last.seen' + type: date + description: Time at which the digest was most recently seen + - name: 'quantile.95' + type: long + description: The 95th percentile of the statement latency, in picoseconds + - name: digest + type: text + description: Performance schema digest + - name: 'count.star' + type: long + description: Number of summarized events + - name: 'avg.timer.wait' + type: long + description: Average wait time of the summarized events that are timed + - name: table_io_waits + type: group + description: Records table I/O waits by index + fields: + - name: object + type: group + fields: + - name: schema + type: keyword + description: Schema name + - name: name + type: keyword + description: Table name + - name: index.name + type: keyword + description: > + Name of the index that was used when the table I/O wait event was recorded. PRIMARY indicates that table I/O + used the primary index. NULL means that table I/O used no index. Inserts are counted against INDEX_NAME = NULL + - name: count.fetch + type: long + description: Number of all fetch operations > 0 diff --git a/metricbeat/module/mysql/performance/manifest.yml b/metricbeat/module/mysql/performance/manifest.yml new file mode 100644 index 00000000000..b88a2694cf7 --- /dev/null +++ b/metricbeat/module/mysql/performance/manifest.yml @@ -0,0 +1,22 @@ +default: true +input: + module: mysql + metricset: query + defaults: + namespace: performance + queries: + - query: > + SELECT digest_text, count_star, avg_timer_wait, max_timer_wait, last_seen, quantile_95 + FROM performance_schema.events_statements_summary_by_digest + ORDER BY avg_timer_wait DESC + LIMIT 10 + query_namespace: events_statements + response_format: table + replace_underscores: true + - query: > + SELECT object_schema, object_name, index_name, count_fetch + FROM performance_schema.table_io_waits_summary_by_index_usage + WHERE count_fetch > 0 + query_namespace: table_io_waits + response_format: table + replace_underscores: true diff --git a/metricbeat/module/mysql/query/_meta/docs.asciidoc b/metricbeat/module/mysql/query/_meta/docs.asciidoc new file mode 100644 index 00000000000..137a026c2e4 --- /dev/null +++ b/metricbeat/module/mysql/query/_meta/docs.asciidoc @@ -0,0 +1 @@ +`query` metricset allows for custom execution of metric related queries in MySQL diff --git a/metricbeat/module/mysql/query/_meta/fields.yml b/metricbeat/module/mysql/query/_meta/fields.yml new file mode 100644 index 00000000000..31215e369a5 --- /dev/null +++ b/metricbeat/module/mysql/query/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: query + type: group + release: beta + description: > + `query` metricset fetches custom queries from the user to a MySQL instance. + fields: diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go new file mode 100644 index 00000000000..24f09218d47 --- /dev/null +++ b/metricbeat/module/mysql/query/query.go @@ -0,0 +1,145 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package status fetches MySQL server status metrics. + +For more information on the query it uses, see: +http://dev.mysql.com/doc/refman/5.7/en/show-status.html +*/ +package query + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/metricbeat/helper/sql" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func init() { + mb.Registry.MustAddMetricSet("mysql", "query", New, + mb.DefaultMetricSet(), + ) +} + +type query struct { + // Namespace for the mysql event. It effectively names the metricset. For example using `performance` will name + // all events `mysql.performance.*` + Namespace string `config:"query_namespace"` + // Query to execute that must return the metrics Metricbeat wants to push to Elasticsearch + Query string `config:"query" validate:"nonzero,required"` + // ResponseFormat has 2 possible values: table and variable. Explained in the SQL helper on Metricbeat + ResponseFormat string `config:"response_format" validate:"nonzero,required"` + // If the query returns keys with underscores like `foo_bar` it will replace that with a `.` to get `foo.bar` JSON key + ReplaceUnderscores bool `config:"replace_underscores"` +} + +// MetricSet for fetching MySQL server status. +type MetricSet struct { + mb.BaseMetricSet + db *sql.DbClient + Config struct { + Queries []query `config:"queries" validate:"nonzero,required"` + Namespace string `config:"namespace" validate:"nonzero,required"` + } +} + +// New creates and returns a new MetricSet instance. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The mysql 'query' metricset is beta.") + + b := &MetricSet{BaseMetricSet: base} + + if err := base.Module().UnpackConfig(&b.Config); err != nil { + return nil, err + } + + return b, nil +} + +// Fetch fetches status messages from a mysql host. +func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { + if m.db == nil { + var err error + m.db, err = sql.NewDBClient("mysql", m.HostData().URI, m.Logger()) + if err != nil { + return errors.Wrap(err, "mysql-status fetch failed") + } + } + + for _, q := range m.Config.Queries { + err := m.fetchQuery(ctx, q, reporter) + if err != nil { + m.Logger().Errorf("error doing query %s", q, err) + } + } + + return nil +} + +func (m *MetricSet) fetchQuery(ctx context.Context, query query, reporter mb.ReporterV2) error { + if query.ResponseFormat == "table" { + mss, err := m.db.FetchTableMode(ctx, query.Query) + if err != nil { + return err + } + + for _, ms := range mss { + event := m.transformMapStrToEvent(query, ms) + reporter.Event(event) + } + } else { + ms, err := m.db.FetchVariableMode(ctx, query.Query) + if err != nil { + return err + } + + event := m.transformMapStrToEvent(query, ms) + reporter.Event(event) + } + + return nil +} + +func (m *MetricSet) transformMapStrToEvent(query query, ms common.MapStr) mb.Event { + event := mb.Event{ModuleFields: common.MapStr{m.Config.Namespace: common.MapStr{}}} + + data := ms + if query.ReplaceUnderscores { + data = sql.ReplaceUnderscores(ms) + } + + if query.Namespace != "" { + event.ModuleFields[m.Config.Namespace] = common.MapStr{query.Namespace: data} + } else { + event.ModuleFields[m.Config.Namespace] = data + } + + return event +} + +// Close closes the database connection and prevents future queries. +func (m *MetricSet) Close() error { + if m.db == nil { + return nil + } + return errors.Wrap(m.db.Close(), "failed to close mysql database client") +} diff --git a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc index f22edf1fa20..31751f264ec 100644 --- a/x-pack/metricbeat/module/sql/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/sql/_meta/docs.asciidoc @@ -1,3 +1,355 @@ -This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query. +The SQL module allows to execute custom queries against an SQL database and store the results to Elasticsearch. +The currently supported databases are the ones already included in Metricbeat, which are: +- PostgreSQL +- MySQL +- Oracle +- Microsoft SQL +- CockroachDB +== Quickstart + +You can setup the module by activating it first running + + metricbeat module enable sql + +Once it is activated, open `modules.d/sql.yml` and fill the required fields. This is an example that captures Innodb related metrics from the result of the query `SHOW GLOBAL STATUS LIKE 'Innodb_system%'` in a MySQL database: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["root:root@tcp(localhost:3306)/ps"] + + driver: "mysql" + sql_query: "SHOW GLOBAL STATUS LIKE 'Innodb_system%'" + sql_response_format: variables +---- + +.SHOW GLOBAL STATUS LIKE 'Innodb_system%' +|==== +|Variable_name|Value + +|Innodb_system_rows_deleted|0 +|Innodb_system_rows_inserted|0 +|Innodb_system_rows_read|5062 +|Innodb_system_rows_updated|315 +|==== + + +Keys in the YAML are defined as follow: + +- `driver`: The drivers currently supported are those which already have a Metricbeat module like `mssql` or `postgres`. +- `sql_query`: Is the single query you want to run +- `sql_response_format`: You have 2 options here: + - `variables`: Expects a table which looks like a key/value result. With 2 columns, left column will be considered a key and the right column the value. This mode generates a single event on each fetch operation. + - `table`: Table mode can contain any number of columns and a single event will be generated for each row. + +Results will be grouped by type in the result event for convenient mapping in Elasticsearch. So `strings` values will be grouped into `sql.strings`, `numeric` into `sql.numeric` and so on and so forth. + +The event generated with the example above looks like this: + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:09:14.407Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "172.18.0.2:3306", + "type": "sql" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 1272810 + }, + "sql": { + "driver": "mysql", + "query": "SHOW GLOBAL STATUS LIKE 'Innodb_system%'", + "metrics": { + "numeric": { + "innodb_system_rows_updated": 315, + "innodb_system_rows_deleted": 0, + "innodb_system_rows_inserted": 0, + "innodb_system_rows_read": 5062 + } + } + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "488431bd-bd3c-4442-ad51-0c50eb555787", + "id": "670ef211-87f0-4f38-8beb-655c377f1629" + } +} +---- + +In this example, we are querying PostgreSQL and generate a "table" result, hence a single event for each row returned + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + + driver: "postgres" + sql_query: "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + sql_response_format: table +---- + +.SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database +|==== +|datid|datname|blks_read|blks_hit|tup_returned|tup_fetched|stats_reset + +|69448|stuff|8652|205976|1484625|53218|2020-06-07 22:50:12 +|13408|postgres|0|0|0|0| +|13407|template0|0|0|0|0| +|==== + +With 3 rows on the table, three events will be generated with the contents of each row. As an example, below you can see the event created for the first row: + +[source,json] +---- +{ + "@timestamp": "2020-06-09T14:47:35.481Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "address": "localhost:5432", + "type": "sql" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "1bffe66d-a1ae-4ed6-985a-fd48548a1971", + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic" + }, + "sql": { + "metrics": { + "numeric": { + "tup_fetched": 53350, + "datid": 69448, + "blks_read": 8652, + "blks_hit": 206501, + "tup_returned": 1.491873e+06 + }, + "string": { + "stats_reset": "2020-06-07T20:50:12.632975Z", + "datname": "stuff" + } + }, + "driver": "postgres", + "query": "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 14076705 + }, + "metricset": { + "name": "query", + "period": 10000 + } +} +---- + + +== More examples + +=== Oracle: + +Get the buffer cache hit ratio: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["oracle://sys:Oradoc_db1@172.17.0.3:1521/ORCLPDB1.localdomain?sysdba=1"] + + driver: "oracle" + sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit Ratio" FROM V$BUFFER_POOL_STATISTICS' + sql_response_format: table +---- + + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:41:02.200Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "sql": { + "metrics": { + "numeric": { + "hit ratio": 0.9742963357937117, + "physical_reads": 17161, + "db_block_gets": 122221, + "consistent_gets": 545427 + }, + "string": { + "name": "DEFAULT" + } + }, + "driver": "oracle", + "query": "SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) \"Hit Ratio\" FROM V$BUFFER_POOL_STATISTICS" + }, + "metricset": { + "period": 10000, + "name": "query" + }, + "service": { + "address": "172.17.0.3:1521", + "type": "sql" + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 39233704 + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + }, + "agent": { + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788" + } +} +---- + +=== MSSQL + +Get the buffer cache hit ratio: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["sqlserver://SA:password@localhost"] + + driver: "mssql" + sql_query: 'SELECT * FROM sys.dm_db_log_space_usage' + sql_response_format: table +---- + +[source,json] +---- +{ + "@timestamp": "2020-06-09T15:39:14.421Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "sql": { + "driver": "mssql", + "query": "SELECT * FROM sys.dm_db_log_space_usage", + "metrics": { + "numeric": { + "log_space_in_bytes_since_last_backup": 524288, + "database_id": 1, + "total_log_size_in_bytes": 2.08896e+06, + "used_log_space_in_bytes": 954368, + "used_log_space_in_percent": 45.686275482177734 + } + } + }, + "event": { + "dataset": "sql.query", + "module": "sql", + "duration": 40750570 + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "172.17.0.2", + "type": "sql" + }, + "agent": { + "id": "670ef211-87f0-4f38-8beb-655c377f1629", + "name": "elastic", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "3da88889-036e-47cb-a88b-275037fa2bc9" + }, + "ecs": { + "version": "1.5.0" + }, + "host": { + "name": "elastic" + } +} +---- + +=== Two or more queries + +If you want to launch two or more queries, you need to specify them with their full configuration for each query. For example: + +.sql.yml +[source,yaml] +---- +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT datid, datname, blks_read, blks_hit, tup_returned, tup_fetched, stats_reset FROM pg_stat_database" + sql_response_format: table + +- module: sql + metricsets: + - query + period: 10s + hosts: ["postgres://postgres:postgres@localhost:5432/stuff?sslmode=disable"] + driver: "postgres" + sql_query: "SELECT * FROM pg_catalog.pg_tables pt WHERE schemaname ='pg_catalog'" + sql_response_format: table +---- diff --git a/x-pack/metricbeat/module/sql/_meta/fields.yml b/x-pack/metricbeat/module/sql/_meta/fields.yml index 844136cedb8..af266792032 100644 --- a/x-pack/metricbeat/module/sql/_meta/fields.yml +++ b/x-pack/metricbeat/module/sql/_meta/fields.yml @@ -25,3 +25,8 @@ object_type: keyword description: > Non-numeric values collected. + - name: metrics.boolean.* + type: object + object_type: keyword + description: > + Boolean values collected. diff --git a/x-pack/metricbeat/module/sql/fields.go b/x-pack/metricbeat/module/sql/fields.go index 8b2c4da414d..e346fd7f8b9 100644 --- a/x-pack/metricbeat/module/sql/fields.go +++ b/x-pack/metricbeat/module/sql/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSql returns asset data. // This is the base64 encoded gzipped contents of module/sql. func AssetSql() string { - return "eJykkkFuwjAQRfc5xRfLSnAAL7rqEiEhDlA59gdcnBjsMW1uX5HEKFWKVFQvv2fmvUy8xImdQrr4ChAnngqL3Xa9qIBIT52oUFN0BVgmE91ZXGgVXisA2G3XaILNnthTzJEJDSU6k7CPoYHuK6wWXevECtg7eptU37xEqxsW+O1Id6bCIYZ8HpNp/bTHRndlvMel9cTuM0Q7yX+RLuetn4GcaCEB/KLJQsiRuGTGbjWj9vH/oNvbiMLquSZ4TyNlcXNquWhzw+jM6mVmEOoPGpnEQ/A+3NqQa8+/6W0Gxv0vjnK0j7WSRNcenrZ6amub0C7Hz8dV+8wHZj+f7HcAAAD//yd20AA=" + return "eJy0k0Fu8jAQhfc5xRPLX4IDZPEvqi4REuIAlWM/wMXJgD2mze0rEoJSpa1AVb18npnvy0Se48C2RDqFAlCvgSVmm/VyVgCRgSaxREU1BeCYbPRH9dKU+F8AwGa9RC0uB2JLtXsm1NTobcI2Sg3TVTijpjKJBbD1DC6VXfMcjak5wC9H2yNL7KLk4zUZ1497XPRnxls8tB7Yvkl0o/wL6eE8dzOQEx1UwHfarITuiVNmbBcTahf/Drq+jBhYHddKCLQ6LG5KHS6aXDN6u/g3MZDqlVZHcR+89LdOchV4n96qZ9z+4lWO7nutpNE3u4etHtraSpr59fNxNiHzLrNKJNA0f6v21EN+1Pr8kj4CAAD//07X+ko=" } diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index d6e91f60eae..5d43d7f785f 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -7,15 +7,13 @@ package query import ( "context" "fmt" - "strconv" - "strings" - "time" "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/metricbeat/helper/sql" "github.com/elastic/beats/v7/metricbeat/mb" ) @@ -81,167 +79,81 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It calls m.fetchTableMode() or m.fetchVariableMode() depending on the response // format of the query. func (m *MetricSet) Fetch(ctx context.Context, report mb.ReporterV2) error { - db, err := m.DB() + db, err := sql.NewDBClient(m.Driver, m.HostData().URI, m.Logger()) if err != nil { return errors.Wrap(err, "error opening connection") } - - rows, err := db.QueryxContext(ctx, m.Query) - if err != nil { - return errors.Wrap(err, "error executing query") - } - defer rows.Close() + defer db.Close() if m.ResponseFormat == tableResponseFormat { - return m.fetchTableMode(rows, report) - } - - return m.fetchVariableMode(rows, report) -} - -// DB gets a client ready to query the database -func (m *MetricSet) DB() (*sqlx.DB, error) { - if m.db == nil { - db, err := sqlx.Open(switchDriverName(m.Driver), m.HostData().URI) - if err != nil { - return nil, errors.Wrap(err, "opening connection") - } - err = db.Ping() + mss, err := db.FetchTableMode(ctx, m.Query) if err != nil { - return nil, errors.Wrap(err, "testing connection") + return err } - m.db = db - } - return m.db, nil -} - -// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. -func (m *MetricSet) fetchTableMode(rows *sqlx.Rows, report mb.ReporterV2) error { - - // Extracted from - // https://stackoverflow.com/questions/23507531/is-golangs-sql-package-incapable-of-ad-hoc-exploratory-queries/23507765#23507765 - cols, err := rows.Columns() - if err != nil { - return errors.Wrap(err, "error getting columns") - } - - for k, v := range cols { - cols[k] = strings.ToLower(v) - } - - vals := make([]interface{}, len(cols)) - for i := 0; i < len(cols); i++ { - vals[i] = new(interface{}) - } - - for rows.Next() { - err = rows.Scan(vals...) - if err != nil { - m.Logger().Debug(errors.Wrap(err, "error trying to scan rows")) - continue - } - - numericMetrics := common.MapStr{} - stringMetrics := common.MapStr{} - - for i := 0; i < len(vals); i++ { - value := getValue(vals[i].(*interface{})) - num, err := strconv.ParseFloat(value, 64) - if err == nil { - numericMetrics[cols[i]] = num - } else { - stringMetrics[cols[i]] = value - } - + for _, ms := range mss { + report.Event(m.getEvent(ms)) } - report.Event(mb.Event{ - RootFields: common.MapStr{ - "sql": common.MapStr{ - "driver": m.Driver, - "query": m.Query, - "metrics": common.MapStr{ - "numeric": numericMetrics, - "string": stringMetrics, - }, - }, - }, - }) + return nil } - if err = rows.Err(); err != nil { - m.Logger().Debug(errors.Wrap(err, "error trying to read rows")) + ms, err := db.FetchVariableMode(ctx, m.Query) + if err != nil { + return err } + report.Event(m.getEvent(ms)) return nil } -// fetchVariableMode scan the rows and publishes the event for querys that return the response in a key/value format. -func (m *MetricSet) fetchVariableMode(rows *sqlx.Rows, report mb.ReporterV2) error { - data := common.MapStr{} - for rows.Next() { - var key string - var val interface{} - err := rows.Scan(&key, &val) - if err != nil { - m.Logger().Debug(errors.Wrap(err, "error trying to scan rows")) - continue - } - - key = strings.ToLower(key) - data[key] = val +func (m *MetricSet) getEvent(ms common.MapStr) mb.Event { + return mb.Event{ + RootFields: common.MapStr{ + "sql": common.MapStr{ + "driver": m.Driver, + "query": m.Query, + "metrics": getMetrics(ms), + }, + }, } +} - if err := rows.Err(); err != nil { - m.Logger().Debug(errors.Wrap(err, "error trying to read rows")) - } +func getMetrics(ms common.MapStr) (ret common.MapStr) { + ret = common.MapStr{} numericMetrics := common.MapStr{} stringMetrics := common.MapStr{} - - for key, value := range data { - value := getValue(&value) - num, err := strconv.ParseFloat(value, 64) - if err == nil { - numericMetrics[key] = num - } else { - stringMetrics[key] = value + boolMetrics := common.MapStr{} + + for k, v := range ms { + switch v.(type) { + case float64: + numericMetrics.Put(k, v) + case string: + stringMetrics.Put(k, v) + case bool: + boolMetrics.Put(k, v) + case nil: + //Ignore because a nil has no data type and thus cannot be indexed + default: + stringMetrics.Put(k, v) } } - report.Event(mb.Event{ - RootFields: common.MapStr{ - "sql": common.MapStr{ - "driver": m.Driver, - "query": m.Query, - "metrics": common.MapStr{ - "numeric": numericMetrics, - "string": stringMetrics, - }, - }, - }, - }) + if len(numericMetrics) > 0 { + ret.Put("numeric", numericMetrics) + } - return nil -} + if len(stringMetrics) > 0 { + ret.Put("string", stringMetrics) + } -func getValue(pval *interface{}) string { - switch v := (*pval).(type) { - case nil: - return "NULL" - case bool: - if v { - return "true" - } - return "false" - case []byte: - return string(v) - case time.Time: - return v.Format(time.RFC3339Nano) - default: - return fmt.Sprint(v) + if len(boolMetrics) > 0 { + ret.Put("bool", boolMetrics) } + + return } // Close closes the connection pool releasing its resources @@ -251,13 +163,3 @@ func (m *MetricSet) Close() error { } return errors.Wrap(m.db.Close(), "closing connection") } - -// switchDriverName switches between driver name and a pretty name for a driver. For example, 'oracle' driver is called -// 'godror' so this detail implementation must be hidden to the user, that should only choose and see 'oracle' as driver -func switchDriverName(d string) string { - if d == "oracle" { - return "godror" - } - - return d -}