From c2db024d015ed46bd022451709b7e1f3128aeb02 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 31 Oct 2016 14:23:53 +0100 Subject: [PATCH] Add username/pass options for PostgreSQL Similar to #2889 but for PostgreSQL. Also adds docs to the Postgres module, which were missing, and adjusted the integration tests to use the username option instead of the full URL. --- CHANGELOG.asciidoc | 1 + metricbeat/docker-compose.yml | 3 +- metricbeat/docs/modules/postgresql.asciidoc | 64 +++++++++++++++- metricbeat/etc/beat.full.yml | 14 +++- metricbeat/metricbeat.full.yml | 14 +++- metricbeat/module/postgresql/_meta/config.yml | 14 +++- .../module/postgresql/_meta/docs.asciidoc | 50 ++++++++++++- .../module/postgresql/activity/activity.go | 22 +++++- .../activity/activity_integration_test.go | 2 + .../module/postgresql/bgwriter/bgwriter.go | 22 +++++- .../bgwriter/bgwriter_integration_test.go | 2 + .../module/postgresql/database/database.go | 21 +++++- .../database/database_integration_test.go | 2 + metricbeat/module/postgresql/postgresql.go | 68 +++++++++++++++++ .../module/postgresql/postgresql_test.go | 74 +++++++++++++++++++ metricbeat/module/postgresql/testing.go | 8 ++ .../tests/system/config/metricbeat.yml.j2 | 8 ++ metricbeat/tests/system/test_postgresql.py | 18 ++++- 18 files changed, 379 insertions(+), 28 deletions(-) create mode 100644 metricbeat/module/postgresql/postgresql_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 04ed2e55324..9d7bd783fca 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v5.0.0...master[Check the HEAD diff] - Add experimental libbeat metricset in the beats module. {pull}2339[2339] - Add experimental docker module. Provided by Ingensi and @douaejeouit based on dockbeat. - Add username and password config options to the MongoDB module. {pull}2889[2889] +- Add username and password config options to the PostgreSQL module. {pull}2889[2890] - Add system core metricset for Windows. {pull}2883}[2883] *Packetbeat* diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 2ce662252ac..5d797610e22 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -24,9 +24,10 @@ beat: - MYSQL_DSN=root:test@tcp(mysql:3306)/ - MYSQL_HOST=mysql - MYSQL_PORT=3306 - - POSTGRESQL_DSN=postgres://postgres@postgresql:5432?sslmode=disable + - POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable - POSTGRESQL_HOST=postgresql - POSTGRESQL_PORT=5432 + - POSTGRESQL_USERNAME=postgres - ZOOKEEPER_HOST=zookeeper - ZOOKEEPER_PORT=2181 - HAPROXY_HOST=haproxy diff --git a/metricbeat/docs/modules/postgresql.asciidoc b/metricbeat/docs/modules/postgresql.asciidoc index af10fbf3d86..f4f3c8634ea 100644 --- a/metricbeat/docs/modules/postgresql.asciidoc +++ b/metricbeat/docs/modules/postgresql.asciidoc @@ -3,10 +3,56 @@ This file is generated! See scripts/docs_collector.py //// [[metricbeat-module-postgresql]] -== postgresql Module +== PostgreSQL Module -This is the postgresql Module. +This module periodically fetches metrics from +https://www.postgresql.org/[PostgreSQL] servers. +[float] +=== Module-Specific Configuration Notes + +When configuring the `hosts` option, you must use Postgres URLs of the following +format: + +----------------------------------- +[postgres://][user:pass@]host[:port][?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"] +---------------------------------------------------------------------- + +WARNING: In case you use username and password in the hosts array, this +information will be sent with each event as part of the `metricset.host` field. +To prevent sending username and password the config options `username` and +`password` can be used. + +[source,yaml] +---- +- module: postgresql + metricsets: ["status"] + hosts: ["postgres://localhost:5432"] + username: root + password: test +---- + +[float] +=== Compatibility + +This module was tested with PostgreSQL 9.5.3 and is expected to work with all +versions >= 9. [float] @@ -33,10 +79,20 @@ metricbeat.modules: #period: 10s # The host must be passed as PostgreSQL DSN. Example: - # postgres://pqgotest:password@localhost:5432?sslmode=disable + # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters - #hosts: ["postgres://postgres@localhost:5432"] + # + # Warning: specifying the user/password in the hosts array is possible + # but it will result in the password being present in the output documents. + # We recommend using the username and password options instead. + #hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass ---- diff --git a/metricbeat/etc/beat.full.yml b/metricbeat/etc/beat.full.yml index 0b868b51ffb..eca495ca3e7 100644 --- a/metricbeat/etc/beat.full.yml +++ b/metricbeat/etc/beat.full.yml @@ -162,10 +162,20 @@ metricbeat.modules: #period: 10s # The host must be passed as PostgreSQL DSN. Example: - # postgres://pqgotest:password@localhost:5432?sslmode=disable + # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters - #hosts: ["postgres://postgres@localhost:5432"] + # + # Warning: specifying the user/password in the hosts array is possible + # but it will result in the password being present in the output documents. + # We recommend using the username and password options instead. + #hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass #-------------------------------- Redis Module ------------------------------- diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 0924561d5cc..00d04552e46 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -162,10 +162,20 @@ metricbeat.modules: #period: 10s # The host must be passed as PostgreSQL DSN. Example: - # postgres://pqgotest:password@localhost:5432?sslmode=disable + # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters - #hosts: ["postgres://postgres@localhost:5432"] + # + # Warning: specifying the user/password in the hosts array is possible + # but it will result in the password being present in the output documents. + # We recommend using the username and password options instead. + #hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass #-------------------------------- Redis Module ------------------------------- diff --git a/metricbeat/module/postgresql/_meta/config.yml b/metricbeat/module/postgresql/_meta/config.yml index d86fe1992db..2998207382b 100644 --- a/metricbeat/module/postgresql/_meta/config.yml +++ b/metricbeat/module/postgresql/_meta/config.yml @@ -13,8 +13,18 @@ #period: 10s # The host must be passed as PostgreSQL DSN. Example: - # postgres://pqgotest:password@localhost:5432?sslmode=disable + # postgres://localhost:5432?sslmode=disable # The available parameters are documented here: # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters - #hosts: ["postgres://postgres@localhost:5432"] + # + # Warning: specifying the user/password in the hosts array is possible + # but it will result in the password being present in the output documents. + # We recommend using the username and password options instead. + #hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass diff --git a/metricbeat/module/postgresql/_meta/docs.asciidoc b/metricbeat/module/postgresql/_meta/docs.asciidoc index accedf4b569..c9a83e2cad0 100644 --- a/metricbeat/module/postgresql/_meta/docs.asciidoc +++ b/metricbeat/module/postgresql/_meta/docs.asciidoc @@ -1,4 +1,50 @@ -== postgresql Module +== PostgreSQL Module -This is the postgresql Module. +This module periodically fetches metrics from +https://www.postgresql.org/[PostgreSQL] servers. +[float] +=== Module-Specific Configuration Notes + +When configuring the `hosts` option, you must use Postgres URLs of the following +format: + +----------------------------------- +[postgres://][user:pass@]host[:port][?options] +----------------------------------- + +The URL can be as simple as: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost"] +---------------------------------------------------------------------- + +Or more complex like: + +[source,yaml] +---------------------------------------------------------------------- +- module: postgresql + hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"] +---------------------------------------------------------------------- + +WARNING: In case you use username and password in the hosts array, this +information will be sent with each event as part of the `metricset.host` field. +To prevent sending username and password the config options `username` and +`password` can be used. + +[source,yaml] +---- +- module: postgresql + metricsets: ["status"] + hosts: ["postgres://localhost:5432"] + username: root + password: test +---- + +[float] +=== Compatibility + +This module was tested with PostgreSQL 9.5.3 and is expected to work with all +versions >= 9. diff --git a/metricbeat/module/postgresql/activity/activity.go b/metricbeat/module/postgresql/activity/activity.go index c6d16f00133..db9046af22b 100644 --- a/metricbeat/module/postgresql/activity/activity.go +++ b/metricbeat/module/postgresql/activity/activity.go @@ -24,6 +24,7 @@ func init() { // MetricSet type defines all fields of the Postgresql MetricSet type MetricSet struct { mb.BaseMetricSet + connectionString string } // New create a new instance of the MetricSet @@ -31,22 +32,35 @@ type MetricSet struct { // configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct{}{} + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + Username string `config:"username"` + Password string `config:"password"` + }{ + Username: "", + Password: "", + } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password, + base.Module().Config().Timeout) + if err != nil { + return nil, err + } + return &MetricSet{ - BaseMetricSet: base, + BaseMetricSet: base, + connectionString: url, }, nil } // Fetch implements the data gathering and data conversion to the right format. func (m *MetricSet) Fetch() ([]common.MapStr, error) { - // TODO: Find a way to pass the timeout - db, err := sql.Open("postgres", m.Host()) + db, err := sql.Open("postgres", m.connectionString) if err != nil { return nil, err } diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go index bd0d8fef565..d8faabc200e 100644 --- a/metricbeat/module/postgresql/activity/activity_integration_test.go +++ b/metricbeat/module/postgresql/activity/activity_integration_test.go @@ -52,5 +52,7 @@ func getConfig() map[string]interface{} { "module": "postgresql", "metricsets": []string{"activity"}, "hosts": []string{postgresql.GetEnvDSN()}, + "username": postgresql.GetEnvUsername(), + "password": postgresql.GetEnvPassword(), } } diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter.go b/metricbeat/module/postgresql/bgwriter/bgwriter.go index 3a18c4bd5eb..b9b0cf95f60 100644 --- a/metricbeat/module/postgresql/bgwriter/bgwriter.go +++ b/metricbeat/module/postgresql/bgwriter/bgwriter.go @@ -24,26 +24,40 @@ func init() { // MetricSet type defines all fields of the MetricSet type MetricSet struct { mb.BaseMetricSet + connectionString string } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - - config := struct{}{} + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + Username string `config:"username"` + Password string `config:"password"` + }{ + Username: "", + Password: "", + } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password, + base.Module().Config().Timeout) + if err != nil { + return nil, err + } + return &MetricSet{ - BaseMetricSet: base, + BaseMetricSet: base, + connectionString: url, }, nil } // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch() (common.MapStr, error) { - db, err := sql.Open("postgres", m.Host()) + db, err := sql.Open("postgres", m.connectionString) if err != nil { return nil, err } diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go index 20cd6a14943..d72a826af2f 100644 --- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go +++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go @@ -54,5 +54,7 @@ func getConfig() map[string]interface{} { "module": "postgresql", "metricsets": []string{"bgwriter"}, "hosts": []string{postgresql.GetEnvDSN()}, + "username": postgresql.GetEnvUsername(), + "password": postgresql.GetEnvPassword(), } } diff --git a/metricbeat/module/postgresql/database/database.go b/metricbeat/module/postgresql/database/database.go index 26817dda1e0..ab33682ef18 100644 --- a/metricbeat/module/postgresql/database/database.go +++ b/metricbeat/module/postgresql/database/database.go @@ -23,26 +23,41 @@ func init() { // MetricSet type defines all fields of the MetricSet type MetricSet struct { mb.BaseMetricSet + connectionString string } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - config := struct{}{} + config := struct { + Hosts []string `config:"hosts" validate:"nonzero,required"` + Username string `config:"username"` + Password string `config:"password"` + }{ + Username: "", + Password: "", + } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password, + base.Module().Config().Timeout) + if err != nil { + return nil, err + } + return &MetricSet{ - BaseMetricSet: base, + BaseMetricSet: base, + connectionString: url, }, nil } // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch() ([]common.MapStr, error) { - db, err := sql.Open("postgres", m.Host()) + db, err := sql.Open("postgres", m.connectionString) if err != nil { return nil, err } diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go index 44a5f58c5be..97ec4481b30 100644 --- a/metricbeat/module/postgresql/database/database_integration_test.go +++ b/metricbeat/module/postgresql/database/database_integration_test.go @@ -54,5 +54,7 @@ func getConfig() map[string]interface{} { "module": "postgresql", "metricsets": []string{"database"}, "hosts": []string{postgresql.GetEnvDSN()}, + "username": postgresql.GetEnvUsername(), + "password": postgresql.GetEnvPassword(), } } diff --git a/metricbeat/module/postgresql/postgresql.go b/metricbeat/module/postgresql/postgresql.go index 9017a6dd6d3..da806450824 100644 --- a/metricbeat/module/postgresql/postgresql.go +++ b/metricbeat/module/postgresql/postgresql.go @@ -5,6 +5,13 @@ package postgresql import ( "database/sql" + "fmt" + "net" + nurl "net/url" + "sort" + "strconv" + "strings" + "time" "github.com/elastic/beats/libbeat/logp" "github.com/pkg/errors" @@ -45,3 +52,64 @@ func QueryStats(db *sql.DB, query string) ([]map[string]interface{}, error) { } return results, nil } + +// ParseURL parses the given URL and overrides the values of username, password and timeout +// if given. Returns a connection string in the form of `user=pass` ready to be passed to the +// sql.Open call. +// Code adapted from the pg driver: https://github.com/lib/pq/blob/master/url.go#L32 +func ParseURL(url, username, password string, timeout time.Duration) (string, error) { + u, err := nurl.Parse(url) + if err != nil { + return "", err + } + + if u.Scheme != "postgres" && u.Scheme != "postgresql" { + return "", fmt.Errorf("invalid connection protocol: %s", u.Scheme) + } + + var kvs []string + escaper := strings.NewReplacer(` `, `\ `, `'`, `\'`, `\`, `\\`) + accrue := func(k, v string) { + if v != "" { + kvs = append(kvs, k+"="+escaper.Replace(v)) + } + } + + if len(username) > 0 { + accrue("user", username) + accrue("password", password) + } else { + if u.User != nil { + v := u.User.Username() + accrue("user", v) + + v, _ = u.User.Password() + accrue("password", v) + } + } + + if host, port, err := net.SplitHostPort(u.Host); err != nil { + accrue("host", u.Host) + } else { + accrue("host", host) + accrue("port", port) + } + + if u.Path != "" { + accrue("dbname", u.Path[1:]) + } + + q := u.Query() + for k := range q { + if k == "connect_timeout" && timeout != 0 { + continue + } + accrue(k, q.Get(k)) + } + if timeout != 0 { + accrue("connect_timeout", strconv.Itoa(int(timeout.Seconds()))) + } + + sort.Strings(kvs) // Makes testing easier (not a performance concern) + return strings.Join(kvs, " "), nil +} diff --git a/metricbeat/module/postgresql/postgresql_test.go b/metricbeat/module/postgresql/postgresql_test.go new file mode 100644 index 00000000000..c6a01c6eb8c --- /dev/null +++ b/metricbeat/module/postgresql/postgresql_test.go @@ -0,0 +1,74 @@ +package postgresql + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParseUrl(t *testing.T) { + tests := []struct { + Name string + URL string + Username string + Password string + Timeout time.Duration + Expected string + }{ + { + Name: "simple test", + URL: "postgres://host1:5432", + Expected: "host=host1 port=5432", + }, + { + Name: "no port", + URL: "postgres://host1", + Expected: "host=host1", + }, + { + Name: "user/pass in URL", + URL: "postgres://user:pass@host1:5432", + Expected: "host=host1 password=pass port=5432 user=user", + }, + { + Name: "user/pass in params", + URL: "postgres://host1:5432", + Username: "user", + Password: "secret", + Expected: "host=host1 password=secret port=5432 user=user", + }, + { + Name: "user/pass override", + URL: "postgres://user1:pass@host1:5432", + Username: "user", + Password: "secret", + Expected: "host=host1 password=secret port=5432 user=user", + }, + { + Name: "timeout no override", + URL: "postgres://host1:5432?connect_timeout=2", + Expected: "connect_timeout=2 host=host1 port=5432", + }, + { + Name: "timeout from param", + URL: "postgres://host1:5432", + Timeout: 3 * time.Second, + Expected: "connect_timeout=3 host=host1 port=5432", + }, + { + Name: "user/pass override, and timeout override", + URL: "postgres://user1:pass@host1:5432?connect_timeout=2", + Username: "user", + Password: "secret", + Timeout: 3 * time.Second, + Expected: "connect_timeout=3 host=host1 password=secret port=5432 user=user", + }, + } + + for _, test := range tests { + url, err := ParseURL(test.URL, test.Username, test.Password, test.Timeout) + assert.NoError(t, err, test.Name) + assert.Equal(t, test.Expected, url, test.Name) + } +} diff --git a/metricbeat/module/postgresql/testing.go b/metricbeat/module/postgresql/testing.go index eabc35b580e..9afe8798358 100644 --- a/metricbeat/module/postgresql/testing.go +++ b/metricbeat/module/postgresql/testing.go @@ -5,3 +5,11 @@ import "os" func GetEnvDSN() string { return os.Getenv("POSTGRESQL_DSN") } + +func GetEnvUsername() string { + return os.Getenv("POSTGRESQL_USERNAME") +} + +func GetEnvPassword() string { + return os.Getenv("POSTGRESQL_PASSWORD") +} diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index bad37431ef7..c7a0e8f47f6 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -14,6 +14,14 @@ metricbeat.modules: {% endfor %} {% endif -%} + {% if m.username -%} + username: {{ m.username }} + {% endif -%} + + {% if m.password -%} + password: {{ m.password }} + {% endif -%} + {% if m.metricsets -%} metricsets: {% for ms in m.metricsets -%} diff --git a/metricbeat/tests/system/test_postgresql.py b/metricbeat/tests/system/test_postgresql.py index bc5e9376be5..ca21bb786e4 100644 --- a/metricbeat/tests/system/test_postgresql.py +++ b/metricbeat/tests/system/test_postgresql.py @@ -18,7 +18,8 @@ def common_checks(self, output): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv("POSTGRESQL_DSN")] + return [os.getenv("POSTGRESQL_DSN")], os.getenv("POSTGRESQL_USERNAME"), \ + os.getenv("POSTGRESQL_PASSWORD") @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @attr('integration') @@ -26,10 +27,13 @@ def test_activity(self): """ PostgreSQL module outputs an event. """ + hosts, username, password = self.get_hosts() self.render_config_template(modules=[{ "name": "postgresql", "metricsets": ["activity"], - "hosts": self.get_hosts(), + "hosts": hosts, + "username": username, + "password": password, "period": "5s" }]) proc = self.start_beat() @@ -50,10 +54,13 @@ def test_database(self): """ PostgreSQL module outputs an event. """ + hosts, username, password = self.get_hosts() self.render_config_template(modules=[{ "name": "postgresql", "metricsets": ["database"], - "hosts": self.get_hosts(), + "hosts": hosts, + "username": username, + "password": password, "period": "5s" }]) proc = self.start_beat() @@ -77,10 +84,13 @@ def test_bgwriter(self): """ PostgreSQL module outputs an event. """ + hosts, username, password = self.get_hosts() self.render_config_template(modules=[{ "name": "postgresql", "metricsets": ["bgwriter"], - "hosts": self.get_hosts(), + "hosts": hosts, + "username": username, + "password": password, "period": "5s" }]) proc = self.start_beat()