Add username/pass options for PostgreSQL (#2890)

Similar to #2889 but for PostgreSQL. Also adds docs to the Postgres module,
which were missing, and adjusted the integration tests to use the username
option instead of the full URL.

CHANGELOG.asciidoc

@@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v5.0.0...master[Check the HEAD diff]
 - Add experimental libbeat metricset in the beats module. {pull}2339[2339]
 - Add experimental docker module. Provided by Ingensi and @douaejeouit based on dockbeat.
 - Add username and password config options to the MongoDB module. {pull}2889[2889]
+- Add username and password config options to the PostgreSQL module. {pull}2889[2890]
 - Add system core metricset for Windows. {pull}2883}[2883]
 
 *Packetbeat*

metricbeat/docker-compose.yml

@@ -24,9 +24,10 @@ beat:
     - MYSQL_DSN=root:test@tcp(mysql:3306)/
     - MYSQL_HOST=mysql
     - MYSQL_PORT=3306
-    - POSTGRESQL_DSN=postgres://postgres@postgresql:5432?sslmode=disable
+    - POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable
     - POSTGRESQL_HOST=postgresql
     - POSTGRESQL_PORT=5432
+    - POSTGRESQL_USERNAME=postgres
     - ZOOKEEPER_HOST=zookeeper
     - ZOOKEEPER_PORT=2181
     - HAPROXY_HOST=haproxy

metricbeat/docs/modules/postgresql.asciidoc

@@ -3,10 +3,56 @@ This file is generated! See scripts/docs_collector.py
 ////
 
 [[metricbeat-module-postgresql]]
-== postgresql Module
+== PostgreSQL Module
 
-This is the postgresql Module.
+This module periodically fetches metrics from
+https://www.postgresql.org/[PostgreSQL] servers.
 
+[float]
+=== Module-Specific Configuration Notes
+
+When configuring the `hosts` option, you must use Postgres URLs of the following
+format:
+
+-----------------------------------
+[postgres://][user:pass@]host[:port][?options]
+-----------------------------------
+
+The URL can be as simple as:
+
+[source,yaml]
+----------------------------------------------------------------------
+- module: postgresql
+  hosts: ["postgres://localhost"]
+----------------------------------------------------------------------
+
+Or more complex like:
+
+[source,yaml]
+----------------------------------------------------------------------
+- module: postgresql
+  hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"]
+----------------------------------------------------------------------
+
+WARNING: In case you use username and password in the hosts array, this
+information will be sent with each event as part of the `metricset.host` field.
+To prevent sending username and password the config options `username` and
+`password` can be used.
+
+[source,yaml]
+----
+- module: postgresql
+  metricsets: ["status"]
+  hosts: ["postgres://localhost:5432"]
+  username: root
+  password: test
+----
+
+[float]
+=== Compatibility
+
+This module was tested with PostgreSQL 9.5.3 and is expected to work with all
+versions >= 9.
 
 
 [float]
@@ -33,10 +79,20 @@ metricbeat.modules:
   #period: 10s
 
   # The host must be passed as PostgreSQL DSN. Example:
-  # postgres://pqgotest:password@localhost:5432?sslmode=disable
+  # postgres://localhost:5432?sslmode=disable
   # The available parameters are documented here:
   # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-  #hosts: ["postgres://postgres@localhost:5432"]
+  #
+  # Warning: specifying the user/password in the hosts array is possible
+  # but it will result in the password being present in the output documents.
+  # We recommend using the username and password options instead.
+  #hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
 
 ----
 

metricbeat/etc/beat.full.yml

@@ -162,10 +162,20 @@ metricbeat.modules:
   #period: 10s
 
   # The host must be passed as PostgreSQL DSN. Example:
-  # postgres://pqgotest:password@localhost:5432?sslmode=disable
+  # postgres://localhost:5432?sslmode=disable
   # The available parameters are documented here:
   # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-  #hosts: ["postgres://postgres@localhost:5432"]
+  #
+  # Warning: specifying the user/password in the hosts array is possible
+  # but it will result in the password being present in the output documents.
+  # We recommend using the username and password options instead.
+  #hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
 
 
 #-------------------------------- Redis Module -------------------------------

metricbeat/metricbeat.full.yml

@@ -162,10 +162,20 @@ metricbeat.modules:
   #period: 10s
 
   # The host must be passed as PostgreSQL DSN. Example:
-  # postgres://pqgotest:password@localhost:5432?sslmode=disable
+  # postgres://localhost:5432?sslmode=disable
   # The available parameters are documented here:
   # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-  #hosts: ["postgres://postgres@localhost:5432"]
+  #
+  # Warning: specifying the user/password in the hosts array is possible
+  # but it will result in the password being present in the output documents.
+  # We recommend using the username and password options instead.
+  #hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
 
 
 #-------------------------------- Redis Module -------------------------------

metricbeat/module/postgresql/_meta/config.yml

@@ -13,8 +13,18 @@
   #period: 10s
 
   # The host must be passed as PostgreSQL DSN. Example:
-  # postgres://pqgotest:password@localhost:5432?sslmode=disable
+  # postgres://localhost:5432?sslmode=disable
   # The available parameters are documented here:
   # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-  #hosts: ["postgres://postgres@localhost:5432"]
+  #
+  # Warning: specifying the user/password in the hosts array is possible
+  # but it will result in the password being present in the output documents.
+  # We recommend using the username and password options instead.
+  #hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
 

metricbeat/module/postgresql/_meta/docs.asciidoc

@@ -1,4 +1,50 @@
-== postgresql Module
+== PostgreSQL Module
 
-This is the postgresql Module.
+This module periodically fetches metrics from
+https://www.postgresql.org/[PostgreSQL] servers.
 
+[float]
+=== Module-Specific Configuration Notes
+
+When configuring the `hosts` option, you must use Postgres URLs of the following
+format:
+
+-----------------------------------
+[postgres://][user:pass@]host[:port][?options]
+-----------------------------------
+
+The URL can be as simple as:
+
+[source,yaml]
+----------------------------------------------------------------------
+- module: postgresql
+  hosts: ["postgres://localhost"]
+----------------------------------------------------------------------
+
+Or more complex like:
+
+[source,yaml]
+----------------------------------------------------------------------
+- module: postgresql
+  hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"]
+----------------------------------------------------------------------
+
+WARNING: In case you use username and password in the hosts array, this
+information will be sent with each event as part of the `metricset.host` field.
+To prevent sending username and password the config options `username` and
+`password` can be used.
+
+[source,yaml]
+----
+- module: postgresql
+  metricsets: ["status"]
+  hosts: ["postgres://localhost:5432"]
+  username: root
+  password: test
+----
+
+[float]
+=== Compatibility
+
+This module was tested with PostgreSQL 9.5.3 and is expected to work with all
+versions >= 9.

metricbeat/module/postgresql/activity/activity.go

@@ -24,29 +24,43 @@ func init() {
 // MetricSet type defines all fields of the Postgresql MetricSet
 type MetricSet struct {
 	mb.BaseMetricSet
+	connectionString string
 }
 
 // New create a new instance of the MetricSet
 // Part of new is also setting up the configuration by processing additional
 // configuration entries if needed.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
-	config := struct{}{}
+	config := struct {
+		Hosts    []string `config:"hosts"    validate:"nonzero,required"`
+		Username string   `config:"username"`
+		Password string   `config:"password"`
+	}{
+		Username: "",
+		Password: "",
+	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
 
+	url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
+		base.Module().Config().Timeout)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
-		BaseMetricSet: base,
+		BaseMetricSet:    base,
+		connectionString: url,
 	}, nil
 }
 
 // Fetch implements the data gathering and data conversion to the right format.
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 
-	// TODO: Find a way to pass the timeout
-	db, err := sql.Open("postgres", m.Host())
+	db, err := sql.Open("postgres", m.connectionString)
 	if err != nil {
 		return nil, err
 	}

metricbeat/module/postgresql/activity/activity_integration_test.go

@@ -52,5 +52,7 @@ func getConfig() map[string]interface{} {
 		"module":     "postgresql",
 		"metricsets": []string{"activity"},
 		"hosts":      []string{postgresql.GetEnvDSN()},
+		"username":   postgresql.GetEnvUsername(),
+		"password":   postgresql.GetEnvPassword(),
 	}
 }

metricbeat/module/postgresql/bgwriter/bgwriter.go

@@ -24,26 +24,40 @@ func init() {
 // MetricSet type defines all fields of the MetricSet
 type MetricSet struct {
 	mb.BaseMetricSet
+	connectionString string
 }
 
 // New create a new instance of the MetricSet
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-
-	config := struct{}{}
+	config := struct {
+		Hosts    []string `config:"hosts"    validate:"nonzero,required"`
+		Username string   `config:"username"`
+		Password string   `config:"password"`
+	}{
+		Username: "",
+		Password: "",
+	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
 
+	url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
+		base.Module().Config().Timeout)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
-		BaseMetricSet: base,
+		BaseMetricSet:    base,
+		connectionString: url,
 	}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right format
 func (m *MetricSet) Fetch() (common.MapStr, error) {
 
-	db, err := sql.Open("postgres", m.Host())
+	db, err := sql.Open("postgres", m.connectionString)
 	if err != nil {
 		return nil, err
 	}

metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go

@@ -54,5 +54,7 @@ func getConfig() map[string]interface{} {
 		"module":     "postgresql",
 		"metricsets": []string{"bgwriter"},
 		"hosts":      []string{postgresql.GetEnvDSN()},
+		"username":   postgresql.GetEnvUsername(),
+		"password":   postgresql.GetEnvPassword(),
 	}
 }

metricbeat/module/postgresql/database/database.go

@@ -23,26 +23,41 @@ func init() {
 // MetricSet type defines all fields of the MetricSet
 type MetricSet struct {
 	mb.BaseMetricSet
+	connectionString string
 }
 
 // New create a new instance of the MetricSet
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
-	config := struct{}{}
+	config := struct {
+		Hosts    []string `config:"hosts"    validate:"nonzero,required"`
+		Username string   `config:"username"`
+		Password string   `config:"password"`
+	}{
+		Username: "",
+		Password: "",
+	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
 
+	url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
+		base.Module().Config().Timeout)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
-		BaseMetricSet: base,
+		BaseMetricSet:    base,
+		connectionString: url,
 	}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right format
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 
-	db, err := sql.Open("postgres", m.Host())
+	db, err := sql.Open("postgres", m.connectionString)
 	if err != nil {
 		return nil, err
 	}

metricbeat/module/postgresql/database/database_integration_test.go

@@ -54,5 +54,7 @@ func getConfig() map[string]interface{} {
 		"module":     "postgresql",
 		"metricsets": []string{"database"},
 		"hosts":      []string{postgresql.GetEnvDSN()},
+		"username":   postgresql.GetEnvUsername(),
+		"password":   postgresql.GetEnvPassword(),
 	}
 }