From 49f4c53ae18b62d902b65b1a2203cd19d3ad3b11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Tue, 20 Jun 2017 16:17:52 +0200
Subject: [PATCH 01/14] Add modules management command

---
 libbeat/cfgfile/glob_manager.go      | 168 +++++++++++++++++++++++++++
 libbeat/cfgfile/glob_manager_test.go |  86 ++++++++++++++
 libbeat/cmd/modules.go               | 104 +++++++++++++++++
 3 files changed, 358 insertions(+)
 create mode 100644 libbeat/cfgfile/glob_manager.go
 create mode 100644 libbeat/cfgfile/glob_manager_test.go
 create mode 100644 libbeat/cmd/modules.go

diff --git a/libbeat/cfgfile/glob_manager.go b/libbeat/cfgfile/glob_manager.go
new file mode 100644
index 00000000000..c8168241085
--- /dev/null
+++ b/libbeat/cfgfile/glob_manager.go
@@ -0,0 +1,168 @@
+package cfgfile
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// GlobManager allows to manage a directory of conf files. Using a glob pattern
+// to match them, this object will allow to switch their state between enabled
+// and disabled
+type GlobManager struct {
+	glob              string
+	enabledExtension  string
+	disabledExtension string
+	files             []*cfgfile
+}
+
+type cfgfile struct {
+	name    string
+	path    string
+	enabled bool
+}
+
+// NewGlobManager takes a glob and enabled/disabled extensions and returns a GlobManager object.
+// Parameters:
+//  - glob - matching conf files (ie: modules.d/*.yml)
+//  - enabledExtension - extension for enabled confs, must match the glob (ie: .yml)
+//  - disabledExtension - extension to append for disabled confs (ie: .disabled)
+func NewGlobManager(glob, enabledExtension, disabledExtension string) (*GlobManager, error) {
+	if !strings.HasSuffix(glob, enabledExtension) {
+		return nil, errors.New("Glob should have the enabledExtension as suffix")
+	}
+
+	g := &GlobManager{
+		glob:              glob,
+		enabledExtension:  enabledExtension,
+		disabledExtension: disabledExtension,
+	}
+	if err := g.load(); err != nil {
+		return nil, err
+	}
+	return g, nil
+}
+
+func (g *GlobManager) load() error {
+	g.files = []*cfgfile{}
+
+	// Load enabled
+	watcher := NewGlobWatcher(g.glob)
+	files, _, err := watcher.Scan()
+	if err != nil {
+		return err
+	}
+
+	for _, path := range files {
+		// Trim cfg file name
+		g.files = append(g.files, &cfgfile{
+			name:    strings.TrimSuffix(filepath.Base(path), g.enabledExtension),
+			enabled: true,
+			path:    path,
+		})
+	}
+
+	// Load disabled
+	watcher = NewGlobWatcher(g.glob + g.disabledExtension)
+	files, _, err = watcher.Scan()
+	if err != nil {
+		return err
+	}
+
+	for _, path := range files {
+		// Trim cfg file name
+		g.files = append(g.files, &cfgfile{
+			name:    strings.TrimSuffix(filepath.Base(path), g.enabledExtension+g.disabledExtension),
+			enabled: false,
+			path:    path,
+		})
+	}
+
+	return nil
+}
+
+// ListEnabled conf files
+func (g *GlobManager) ListEnabled() []string {
+	names := []string{}
+
+	for _, file := range g.files {
+		if file.enabled {
+			names = append(names, file.name)
+		}
+	}
+
+	return names
+}
+
+// ListDisabled conf files
+func (g *GlobManager) ListDisabled() []string {
+	names := []string{}
+
+	for _, file := range g.files {
+		if !file.enabled {
+			names = append(names, file.name)
+		}
+	}
+
+	return names
+}
+
+// Enabled returns true if given conf file is enabled
+func (g *GlobManager) Enabled(name string) bool {
+	for _, file := range g.files {
+		if name == file.name {
+			return file.enabled
+		}
+	}
+	return false
+}
+
+// Exists return true if the given conf exists (enabled or disabled)
+func (g *GlobManager) Exists(name string) bool {
+	for _, file := range g.files {
+		if name == file.name {
+			return true
+		}
+	}
+	return false
+}
+
+// Enable given conf file, does nothing if it's enabled already
+func (g *GlobManager) Enable(name string) error {
+	for _, file := range g.files {
+		if name == file.name {
+			if !file.enabled {
+				newPath := strings.TrimSuffix(file.path, g.disabledExtension)
+				if err := os.Rename(file.path, newPath); err != nil {
+					return errors.Wrap(err, "enable failed")
+				}
+				file.enabled = true
+				file.path = newPath
+			}
+			return nil
+		}
+	}
+
+	return errors.Errorf("module %s not found", name)
+}
+
+// Disable given conf file, does nothing if it's disabled already
+func (g *GlobManager) Disable(name string) error {
+	for _, file := range g.files {
+		if name == file.name {
+			if file.enabled {
+				newPath := file.path + g.disabledExtension
+				if err := os.Rename(file.path, newPath); err != nil {
+					return errors.Wrap(err, "disable failed")
+				}
+				file.enabled = false
+				file.path = newPath
+			}
+			return nil
+		}
+	}
+
+	return errors.Errorf("module %s not found", name)
+}
diff --git a/libbeat/cfgfile/glob_manager_test.go b/libbeat/cfgfile/glob_manager_test.go
new file mode 100644
index 00000000000..8d97ad89998
--- /dev/null
+++ b/libbeat/cfgfile/glob_manager_test.go
@@ -0,0 +1,86 @@
+package cfgfile
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGlobManagerInit(t *testing.T) {
+	// Wrong settings return error
+	manager, err := NewGlobManager("dir/*.yml", ".noyml", ".disabled")
+	assert.Error(t, err)
+	assert.Nil(t, manager)
+}
+
+func TestGlobManager(t *testing.T) {
+	// Create random temp directory
+	dir, err := ioutil.TempDir("", "glob_manager")
+	defer os.RemoveAll(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Prepare scenario:
+	content := []byte("test\n")
+	err = ioutil.WriteFile(dir+"/config1.yml", content, 0644)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(dir+"/config2.yml", content, 0644)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(dir+"/config3.yml.disabled", content, 0644)
+	assert.NoError(t, err)
+
+	// Init Glob Manager
+	glob := dir + "/*.yml"
+	manager, err := NewGlobManager(glob, ".yml", ".disabled")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.True(t, manager.Exists("config1"))
+	assert.True(t, manager.Exists("config2"))
+	assert.True(t, manager.Exists("config3"))
+	assert.False(t, manager.Exists("config4"))
+
+	assert.True(t, manager.Enabled("config1"))
+	assert.True(t, manager.Enabled("config2"))
+	assert.False(t, manager.Enabled("config3"))
+
+	assert.Equal(t, len(manager.ListEnabled()), 2)
+	assert.Equal(t, len(manager.ListDisabled()), 1)
+
+	// Test disable
+	if err = manager.Disable("config2"); err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, len(manager.ListEnabled()), 1)
+	assert.Equal(t, len(manager.ListDisabled()), 2)
+
+	assert.Equal(t, manager.ListEnabled(), []string{"config1"})
+
+	// Test enable
+	if err = manager.Enable("config3"); err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, len(manager.ListEnabled()), 2)
+	assert.Equal(t, len(manager.ListDisabled()), 1)
+
+	assert.Equal(t, manager.ListDisabled(), []string{"config2"})
+
+	// Check correct files layout:
+	files, err := filepath.Glob(dir + "/*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, files, []string{
+		dir + "/config1.yml",
+		dir + "/config2.yml.disabled",
+		dir + "/config3.yml",
+	})
+}
diff --git a/libbeat/cmd/modules.go b/libbeat/cmd/modules.go
new file mode 100644
index 00000000000..a660e9973d8
--- /dev/null
+++ b/libbeat/cmd/modules.go
@@ -0,0 +1,104 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+type ModulesManager interface {
+	ListEnabled() []string
+	ListDisabled() []string
+	Exists(name string) bool
+	Enabled(name string) bool
+	Enable(name string) error
+	Disable(name string) error
+}
+
+// GenModulesCmd initializes a command to manage a modules.d folder, it offers
+// list, enable and siable actions
+func GenModulesCmd(name, version string, modules ModulesManager) *cobra.Command {
+	modulesCmd := cobra.Command{
+		Use:   "modules",
+		Short: "Manage configured modules",
+	}
+
+	modulesCmd.AddCommand(genListModulesCmd(modules))
+	modulesCmd.AddCommand(genEnableModulesCmd(modules))
+	modulesCmd.AddCommand(genDisableModulesCmd(modules))
+
+	return &modulesCmd
+}
+
+func genListModulesCmd(modules ModulesManager) *cobra.Command {
+	return &cobra.Command{
+		Use:   "list",
+		Short: "List modules",
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Println("Enabled:")
+			for _, module := range modules.ListEnabled() {
+				fmt.Println(module)
+			}
+
+			fmt.Println("\nDisabled:")
+			for _, module := range modules.ListDisabled() {
+				fmt.Println(module)
+			}
+		},
+	}
+}
+
+func genEnableModulesCmd(modules ModulesManager) *cobra.Command {
+	return &cobra.Command{
+		Use:   "enable MODULE...",
+		Short: "Enable one or more given modules",
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, module := range args {
+				if !modules.Exists(module) {
+					fmt.Printf("Module %s doesn't exists!\n", module)
+					os.Exit(1)
+				}
+
+				if modules.Enabled(module) {
+					fmt.Printf("Module %s is already enabled\n", module)
+					continue
+				}
+
+				if err := modules.Enable(module); err != nil {
+					fmt.Fprintf(os.Stderr, "There was an error enabling module %s: %s\n", module, err)
+					os.Exit(1)
+				}
+
+				fmt.Printf("Enabled %s\n", module)
+			}
+		},
+	}
+}
+
+func genDisableModulesCmd(modules ModulesManager) *cobra.Command {
+	return &cobra.Command{
+		Use:   "disable MODULE...",
+		Short: "Disable one or more given modules",
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, module := range args {
+				if !modules.Exists(module) {
+					fmt.Fprintf(os.Stderr, "Module %s doesn't exists!\n", module)
+					os.Exit(1)
+				}
+
+				if !modules.Enabled(module) {
+					fmt.Fprintf(os.Stderr, "Module %s is already disabled\n", module)
+					continue
+				}
+
+				if err := modules.Disable(module); err != nil {
+					fmt.Fprintf(os.Stderr, "There was an error disabling module %s: %s\n", module, err)
+					os.Exit(1)
+				}
+
+				fmt.Printf("Disabled %s\n", module)
+			}
+		},
+	}
+}

From 5afd58ce5f52848f3d56b890517a68e5151306db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Tue, 20 Jun 2017 11:59:18 +0200
Subject: [PATCH 02/14] Unify config reloader usage

---
 libbeat/cfgfile/reload.go       | 10 ++++++----
 metricbeat/beater/config.go     |  2 +-
 metricbeat/beater/metricbeat.go |  7 +++----
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/libbeat/cfgfile/reload.go b/libbeat/cfgfile/reload.go
index 79cf7afed1f..e382b33c7c8 100644
--- a/libbeat/cfgfile/reload.go
+++ b/libbeat/cfgfile/reload.go
@@ -12,7 +12,7 @@ import (
 )
 
 var (
-	DefaultReloadConfig = ReloadConfig{
+	DefaultDynamicConfig = DynamicConfig{
 		Reload: Reload{
 			Period:  10 * time.Second,
 			Enabled: false,
@@ -27,7 +27,9 @@ var (
 	moduleRunning = monitoring.NewInt(nil, "libbeat.config.module.running")
 )
 
-type ReloadConfig struct {
+// DynamicConfig loads config files from a given path, allowing to reload new changes
+// while running the beat
+type DynamicConfig struct {
 	// If path is a relative path, it is relative to the ${path.config}
 	Path   string `config:"path"`
 	Reload Reload `config:"reload"`
@@ -51,7 +53,7 @@ type Runner interface {
 // Reloader is used to register and reload modules
 type Reloader struct {
 	registry *Registry
-	config   ReloadConfig
+	config   DynamicConfig
 	done     chan struct{}
 	wg       sync.WaitGroup
 }
@@ -59,7 +61,7 @@ type Reloader struct {
 // NewReloader creates new Reloader instance for the given config
 func NewReloader(cfg *common.Config) *Reloader {
 
-	config := DefaultReloadConfig
+	config := DefaultDynamicConfig
 	cfg.Unpack(&config)
 
 	return &Reloader{
diff --git a/metricbeat/beater/config.go b/metricbeat/beater/config.go
index 6e7e06ac632..7c2f9f3e9d3 100644
--- a/metricbeat/beater/config.go
+++ b/metricbeat/beater/config.go
@@ -10,7 +10,7 @@ import (
 type Config struct {
 	// Modules is a list of module specific configuration data.
 	Modules       []*common.Config `config:"modules"`
-	ReloadModules *common.Config   `config:"config.modules"`
+	ConfigModules *common.Config   `config:"config.modules"`
 	MaxStartDelay time.Duration    `config:"max_start_delay"` // Upper bound on the random startup delay for metricsets (use 0 to disable startup delay).
 }
 
diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go
index d87e82abf28..4140f8cb04e 100644
--- a/metricbeat/beater/metricbeat.go
+++ b/metricbeat/beater/metricbeat.go
@@ -38,7 +38,7 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 	modules, err := module.NewWrappers(config.MaxStartDelay, config.Modules, mb.Registry)
 	if err != nil {
 		// Empty config is fine if dynamic config is enabled
-		if !config.ReloadModules.Enabled() {
+		if !config.ConfigModules.Enabled() {
 			return nil, err
 		} else if err != mb.ErrEmptyConfig && err != mb.ErrAllModulesDisabled {
 			return nil, err
@@ -72,9 +72,8 @@ func (bt *Metricbeat) Run(b *beat.Beat) error {
 		}()
 	}
 
-	if bt.config.ReloadModules.Enabled() {
-		logp.Beta("feature dynamic configuration reloading is enabled.")
-		moduleReloader := cfgfile.NewReloader(bt.config.ReloadModules)
+	if bt.config.ConfigModules.Enabled() {
+		moduleReloader := cfgfile.NewReloader(bt.config.ConfigModules)
 		factory := module.NewFactory(bt.config.MaxStartDelay, b.Publisher)
 
 		go moduleReloader.Run(factory)

From dd9150c4692b7278e8965970df8752ebda01f772 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Mon, 19 Jun 2017 16:17:11 +0200
Subject: [PATCH 03/14] Move module settings to `modules.d` folder

We only want to use modules.d layout for `metricbeat.yml`, while keeping
`metricbeat.reference.yml` as a single file with all config options
---
 metricbeat/Makefile                           | 10 ++++--
 metricbeat/_meta/setup.yml                    | 12 +++++++
 metricbeat/docs/modules/apache.asciidoc       |  1 -
 metricbeat/docs/modules/ceph.asciidoc         |  1 -
 metricbeat/docs/modules/couchbase.asciidoc    |  1 -
 metricbeat/docs/modules/docker.asciidoc       |  1 -
 metricbeat/docs/modules/dropwizard.asciidoc   |  1 -
 .../docs/modules/elasticsearch.asciidoc       |  3 --
 metricbeat/docs/modules/golang.asciidoc       |  1 -
 metricbeat/docs/modules/haproxy.asciidoc      |  1 -
 metricbeat/docs/modules/http.asciidoc         |  1 -
 metricbeat/docs/modules/jolokia.asciidoc      |  1 -
 metricbeat/docs/modules/kafka.asciidoc        |  1 -
 metricbeat/docs/modules/kibana.asciidoc       |  1 -
 metricbeat/docs/modules/kubernetes.asciidoc   |  2 --
 metricbeat/docs/modules/memcached.asciidoc    |  2 --
 metricbeat/docs/modules/mongodb.asciidoc      |  1 -
 metricbeat/docs/modules/mysql.asciidoc        |  1 -
 metricbeat/docs/modules/nginx.asciidoc        |  1 -
 metricbeat/docs/modules/php_fpm.asciidoc      |  1 -
 metricbeat/docs/modules/postgresql.asciidoc   |  2 --
 metricbeat/docs/modules/prometheus.asciidoc   |  1 -
 metricbeat/docs/modules/rabbitmq.asciidoc     |  1 -
 metricbeat/docs/modules/redis.asciidoc        |  1 -
 metricbeat/docs/modules/system.asciidoc       |  3 +-
 metricbeat/docs/modules/vsphere.asciidoc      |  1 -
 metricbeat/docs/modules/windows.asciidoc      |  1 -
 metricbeat/docs/modules/zookeeper.asciidoc    |  1 -
 metricbeat/metricbeat.reference.yml           | 29 ---------------
 metricbeat/metricbeat.yml                     | 36 +++++--------------
 .../module/apache/_meta/config.reference.yml  |  1 -
 metricbeat/module/apache/_meta/config.yml     |  1 -
 metricbeat/module/ceph/_meta/config.yml       |  1 -
 metricbeat/module/couchbase/_meta/config.yml  |  1 -
 metricbeat/module/docker/_meta/config.yml     |  1 -
 metricbeat/module/dropwizard/_meta/config.yml |  1 -
 .../module/elasticsearch/_meta/config.yml     |  3 --
 metricbeat/module/golang/_meta/config.yml     |  1 -
 metricbeat/module/haproxy/_meta/config.yml    |  1 -
 metricbeat/module/http/_meta/config.yml       |  1 -
 metricbeat/module/jolokia/_meta/config.yml    |  1 -
 metricbeat/module/kafka/_meta/config.yml      |  1 -
 metricbeat/module/kibana/_meta/config.yml     |  1 -
 metricbeat/module/kubernetes/_meta/config.yml |  2 --
 metricbeat/module/memcached/_meta/config.yml  |  2 --
 metricbeat/module/mongodb/_meta/config.yml    |  1 -
 metricbeat/module/mysql/_meta/config.yml      |  1 -
 metricbeat/module/nginx/_meta/config.yml      |  1 -
 metricbeat/module/php_fpm/_meta/config.yml    |  1 -
 metricbeat/module/postgresql/_meta/config.yml |  2 --
 metricbeat/module/prometheus/_meta/config.yml |  1 -
 metricbeat/module/rabbitmq/_meta/config.yml   |  1 -
 metricbeat/module/redis/_meta/config.yml      |  1 -
 metricbeat/module/system/_meta/config.yml     |  3 +-
 metricbeat/module/vsphere/_meta/config.yml    |  1 -
 metricbeat/module/windows/_meta/config.yml    |  1 -
 metricbeat/module/zookeeper/_meta/config.yml  |  1 -
 metricbeat/modules.d/aerospike.yml.disabled   |  5 +++
 metricbeat/modules.d/apache.yml.disabled      |  6 ++++
 metricbeat/modules.d/ceph.yml.disabled        |  4 +++
 metricbeat/modules.d/couchbase.yml.disabled   |  4 +++
 metricbeat/modules.d/docker.yml.disabled      | 10 ++++++
 metricbeat/modules.d/dropwizard.yml.disabled  |  6 ++++
 .../modules.d/elasticsearch.yml.disabled      |  4 +++
 metricbeat/modules.d/golang.yml.disabled      |  8 +++++
 metricbeat/modules.d/haproxy.yml.disabled     |  4 +++
 metricbeat/modules.d/http.yml.disabled        | 10 ++++++
 metricbeat/modules.d/jolokia.yml.disabled     |  9 +++++
 metricbeat/modules.d/kafka.yml.disabled       | 25 +++++++++++++
 metricbeat/modules.d/kibana.yml.disabled      |  4 +++
 metricbeat/modules.d/kubernetes.yml.disabled  | 28 +++++++++++++++
 metricbeat/modules.d/memcached.yml.disabled   |  4 +++
 metricbeat/modules.d/mysql.yml.disabled       | 17 +++++++++
 metricbeat/modules.d/nginx.yml.disabled       |  9 +++++
 metricbeat/modules.d/php_fpm.yml.disabled     |  5 +++
 metricbeat/modules.d/postgresql.yml.disabled  | 24 +++++++++++++
 metricbeat/modules.d/prometheus.yml.disabled  |  6 ++++
 metricbeat/modules.d/rabbitmq.yml.disabled    |  7 ++++
 metricbeat/modules.d/redis.yml.disabled       | 29 +++++++++++++++
 metricbeat/modules.d/system.yml               | 24 +++++++++++++
 metricbeat/modules.d/vsphere.yml.disabled     |  9 +++++
 metricbeat/modules.d/windows.yml.disabled     |  4 +++
 metricbeat/modules.d/zookeeper.yml.disabled   |  4 +++
 83 files changed, 298 insertions(+), 125 deletions(-)
 create mode 100644 metricbeat/modules.d/aerospike.yml.disabled
 create mode 100644 metricbeat/modules.d/apache.yml.disabled
 create mode 100644 metricbeat/modules.d/ceph.yml.disabled
 create mode 100644 metricbeat/modules.d/couchbase.yml.disabled
 create mode 100644 metricbeat/modules.d/docker.yml.disabled
 create mode 100644 metricbeat/modules.d/dropwizard.yml.disabled
 create mode 100644 metricbeat/modules.d/elasticsearch.yml.disabled
 create mode 100644 metricbeat/modules.d/golang.yml.disabled
 create mode 100644 metricbeat/modules.d/haproxy.yml.disabled
 create mode 100644 metricbeat/modules.d/http.yml.disabled
 create mode 100644 metricbeat/modules.d/jolokia.yml.disabled
 create mode 100644 metricbeat/modules.d/kafka.yml.disabled
 create mode 100644 metricbeat/modules.d/kibana.yml.disabled
 create mode 100644 metricbeat/modules.d/kubernetes.yml.disabled
 create mode 100644 metricbeat/modules.d/memcached.yml.disabled
 create mode 100644 metricbeat/modules.d/mysql.yml.disabled
 create mode 100644 metricbeat/modules.d/nginx.yml.disabled
 create mode 100644 metricbeat/modules.d/php_fpm.yml.disabled
 create mode 100644 metricbeat/modules.d/postgresql.yml.disabled
 create mode 100644 metricbeat/modules.d/prometheus.yml.disabled
 create mode 100644 metricbeat/modules.d/rabbitmq.yml.disabled
 create mode 100644 metricbeat/modules.d/redis.yml.disabled
 create mode 100644 metricbeat/modules.d/system.yml
 create mode 100644 metricbeat/modules.d/vsphere.yml.disabled
 create mode 100644 metricbeat/modules.d/windows.yml.disabled
 create mode 100644 metricbeat/modules.d/zookeeper.yml.disabled

diff --git a/metricbeat/Makefile b/metricbeat/Makefile
index e38822a820f..ae419456b31 100644
--- a/metricbeat/Makefile
+++ b/metricbeat/Makefile
@@ -47,10 +47,14 @@ collect-docs: python-env
 .PHONY: configs
 configs: python-env
 	@cp ${ES_BEATS}/metricbeat/_meta/common.yml _meta/beat.yml
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml
 	@cat ${ES_BEATS}/metricbeat/_meta/setup.yml >> _meta/beat.yml
-	@cp ${ES_BEATS}/metricbeat/_meta/common.reference.yml _meta/beat.reference.yml
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
+	@cat ${ES_BEATS}/metricbeat/_meta/common.reference.yml > _meta/beat.reference.yml
+	@. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
+	@rm -rf modules.d && mkdir -p modules.d
+	@for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
+	@chmod go-w modules.d/*
+	# Enable system by default:
+	@mv modules.d/system.yml.disabled modules.d/system.yml
 
 # Generates imports for all modules and metricsets
 .PHONY: imports
diff --git a/metricbeat/_meta/setup.yml b/metricbeat/_meta/setup.yml
index c18143a7cd4..611f841e45f 100644
--- a/metricbeat/_meta/setup.yml
+++ b/metricbeat/_meta/setup.yml
@@ -1,3 +1,15 @@
+
+#==========================  Modules configuration ============================
+metricbeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  reload.period: 10s
+
 #==================== Elasticsearch template setting ==========================
 setup.template.settings:
   index.number_of_shards: 1
diff --git a/metricbeat/docs/modules/apache.asciidoc b/metricbeat/docs/modules/apache.asciidoc
index b9ca1e094b5..b22ac468e35 100644
--- a/metricbeat/docs/modules/apache.asciidoc
+++ b/metricbeat/docs/modules/apache.asciidoc
@@ -34,7 +34,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: apache
   metricsets: ["status"]
-  enabled: false
   period: 10s
 
   # Apache hosts
diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc
index af41881c1dd..6606fc8d847 100644
--- a/metricbeat/docs/modules/ceph.asciidoc
+++ b/metricbeat/docs/modules/ceph.asciidoc
@@ -22,7 +22,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: ceph
   metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"]
-  enabled: false
   period: 10s
   hosts: ["localhost:5000"]
 ----
diff --git a/metricbeat/docs/modules/couchbase.asciidoc b/metricbeat/docs/modules/couchbase.asciidoc
index e6a231b398e..cc98537b568 100644
--- a/metricbeat/docs/modules/couchbase.asciidoc
+++ b/metricbeat/docs/modules/couchbase.asciidoc
@@ -22,7 +22,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: couchbase
   metricsets: ["bucket", "cluster", "node"]
-  enabled: false
   period: 10s
   hosts: ["localhost:8091"]
 ----
diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc
index 8ac346d3685..4a9b7e33800 100644
--- a/metricbeat/docs/modules/docker.asciidoc
+++ b/metricbeat/docs/modules/docker.asciidoc
@@ -33,7 +33,6 @@ metricbeat.modules:
 - module: docker
   metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"]
   hosts: ["unix:///var/run/docker.sock"]
-  enabled: false
   period: 10s
 
   # To connect to Docker over TLS you must specify a client and CA certificate.
diff --git a/metricbeat/docs/modules/dropwizard.asciidoc b/metricbeat/docs/modules/dropwizard.asciidoc
index 970dbd81206..91ef03032ae 100644
--- a/metricbeat/docs/modules/dropwizard.asciidoc
+++ b/metricbeat/docs/modules/dropwizard.asciidoc
@@ -20,7 +20,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: dropwizard
   metricsets: ["collector"]
-  enabled: false
   period: 10s
   hosts: ["localhost:8080"]
   metrics_path: /metrics/metrics
diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc
index 3de0f509783..0ae9c1be33c 100644
--- a/metricbeat/docs/modules/elasticsearch.asciidoc
+++ b/metricbeat/docs/modules/elasticsearch.asciidoc
@@ -19,11 +19,8 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: elasticsearch
   metricsets: ["node", "node_stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9200"]
-
-
 ----
 
 [float]
diff --git a/metricbeat/docs/modules/golang.asciidoc b/metricbeat/docs/modules/golang.asciidoc
index 2a6d741fd00..57663c5a0cf 100644
--- a/metricbeat/docs/modules/golang.asciidoc
+++ b/metricbeat/docs/modules/golang.asciidoc
@@ -19,7 +19,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: golang
   metricsets: ["expvar","heap"]
-  enabled: false
   period: 10s
   hosts: ["localhost:6060"]
   heap.path: "/debug/vars"
diff --git a/metricbeat/docs/modules/haproxy.asciidoc b/metricbeat/docs/modules/haproxy.asciidoc
index 3ed714f7323..b45c528f88c 100644
--- a/metricbeat/docs/modules/haproxy.asciidoc
+++ b/metricbeat/docs/modules/haproxy.asciidoc
@@ -32,7 +32,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: haproxy
   metricsets: ["info", "stat"]
-  enabled: false
   period: 10s
   hosts: ["tcp://127.0.0.1:14567"]
 ----
diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc
index 86d929ba07d..3de4f5da909 100644
--- a/metricbeat/docs/modules/http.asciidoc
+++ b/metricbeat/docs/modules/http.asciidoc
@@ -25,7 +25,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: http
   metricsets: ["json"]
-  enabled: false
   period: 10s
   hosts: ["localhost:80"]
   namespace: "json_namespace"
diff --git a/metricbeat/docs/modules/jolokia.asciidoc b/metricbeat/docs/modules/jolokia.asciidoc
index f68ff302f5d..10ade8625fd 100644
--- a/metricbeat/docs/modules/jolokia.asciidoc
+++ b/metricbeat/docs/modules/jolokia.asciidoc
@@ -22,7 +22,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: jolokia
   metricsets: ["jmx"]
-  enabled: false
   period: 10s
   hosts: ["localhost"]
   namespace: "metrics"
diff --git a/metricbeat/docs/modules/kafka.asciidoc b/metricbeat/docs/modules/kafka.asciidoc
index 058a3b444a7..6541affa237 100644
--- a/metricbeat/docs/modules/kafka.asciidoc
+++ b/metricbeat/docs/modules/kafka.asciidoc
@@ -22,7 +22,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: kafka
   metricsets: ["partition"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9092"]
 
diff --git a/metricbeat/docs/modules/kibana.asciidoc b/metricbeat/docs/modules/kibana.asciidoc
index ff90a453ce2..ee6e8f82619 100644
--- a/metricbeat/docs/modules/kibana.asciidoc
+++ b/metricbeat/docs/modules/kibana.asciidoc
@@ -19,7 +19,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: kibana
   metricsets: ["status"]
-  enabled: false
   period: 10s
   hosts: ["localhost:5601"]
 ----
diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc
index de3a789f9c4..f142f015204 100644
--- a/metricbeat/docs/modules/kubernetes.asciidoc
+++ b/metricbeat/docs/modules/kubernetes.asciidoc
@@ -26,7 +26,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 # Node metrics, from kubelet:
 - module: kubernetes
-  enabled: false
   metricsets:
     - node
     - system
@@ -53,7 +52,6 @@ metricbeat.modules:
   enabled: false
   metricsets:
     - event
-  in_cluster: true
 ----
 
 [float]
diff --git a/metricbeat/docs/modules/memcached.asciidoc b/metricbeat/docs/modules/memcached.asciidoc
index b24dd5b9c7d..cf17cb99626 100644
--- a/metricbeat/docs/modules/memcached.asciidoc
+++ b/metricbeat/docs/modules/memcached.asciidoc
@@ -20,10 +20,8 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: memcached
   metricsets: ["stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:11211"]
-
 ----
 
 [float]
diff --git a/metricbeat/docs/modules/mongodb.asciidoc b/metricbeat/docs/modules/mongodb.asciidoc
index 7d625416ceb..c6c5cd9293c 100644
--- a/metricbeat/docs/modules/mongodb.asciidoc
+++ b/metricbeat/docs/modules/mongodb.asciidoc
@@ -65,7 +65,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: mongodb
   metricsets: ["dbstats", "status"]
-  enabled: false
   period: 10s
 
   # The hosts must be passed as MongoDB URLs in the format:
diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc
index e84702b1863..39c2f124f9e 100644
--- a/metricbeat/docs/modules/mysql.asciidoc
+++ b/metricbeat/docs/modules/mysql.asciidoc
@@ -48,7 +48,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: mysql
   metricsets: ["status"]
-  enabled: false
   period: 10s
 
   # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/"
diff --git a/metricbeat/docs/modules/nginx.asciidoc b/metricbeat/docs/modules/nginx.asciidoc
index 1c280f4386a..f240267f1ad 100644
--- a/metricbeat/docs/modules/nginx.asciidoc
+++ b/metricbeat/docs/modules/nginx.asciidoc
@@ -26,7 +26,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: nginx
   metricsets: ["stubstatus"]
-  enabled: false
   period: 10s
 
   # Nginx hosts
diff --git a/metricbeat/docs/modules/php_fpm.asciidoc b/metricbeat/docs/modules/php_fpm.asciidoc
index 40fce8314e0..5ece5e82472 100644
--- a/metricbeat/docs/modules/php_fpm.asciidoc
+++ b/metricbeat/docs/modules/php_fpm.asciidoc
@@ -46,7 +46,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: php_fpm
   metricsets: ["pool"]
-  enabled: false
   period: 10s
   status_path: "/status"
   hosts: ["localhost:8080"]
diff --git a/metricbeat/docs/modules/postgresql.asciidoc b/metricbeat/docs/modules/postgresql.asciidoc
index 1c8876ff04c..7adc4f2cc27 100644
--- a/metricbeat/docs/modules/postgresql.asciidoc
+++ b/metricbeat/docs/modules/postgresql.asciidoc
@@ -74,7 +74,6 @@ metricbeat.modules:
     # Stats about every PostgreSQL process
     - activity
 
-  enabled: false
   period: 10s
 
   # The host must be passed as PostgreSQL URL. Example:
@@ -88,7 +87,6 @@ metricbeat.modules:
 
   # Password to use when connecting to PostgreSQL. Empty by default.
   #password: pass
-
 ----
 
 [float]
diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc
index b25d9256a82..d84e2dc5f65 100644
--- a/metricbeat/docs/modules/prometheus.asciidoc
+++ b/metricbeat/docs/modules/prometheus.asciidoc
@@ -22,7 +22,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: prometheus
   metricsets: ["stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9090"]
   metrics_path: /metrics
diff --git a/metricbeat/docs/modules/rabbitmq.asciidoc b/metricbeat/docs/modules/rabbitmq.asciidoc
index da2db7b8428..4eb9c1617d8 100644
--- a/metricbeat/docs/modules/rabbitmq.asciidoc
+++ b/metricbeat/docs/modules/rabbitmq.asciidoc
@@ -20,7 +20,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: rabbitmq
   metricsets: ["node"]
-  enabled: false
   period: 10s
   hosts: ["localhost:15672"]
 
diff --git a/metricbeat/docs/modules/redis.asciidoc b/metricbeat/docs/modules/redis.asciidoc
index 5c3923d4c0a..8cbd57ce406 100644
--- a/metricbeat/docs/modules/redis.asciidoc
+++ b/metricbeat/docs/modules/redis.asciidoc
@@ -42,7 +42,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: redis
   metricsets: ["info", "keyspace"]
-  enabled: false
   period: 10s
 
   # Redis hosts
diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc
index db0c104558f..e9ae39dae7c 100644
--- a/metricbeat/docs/modules/system.asciidoc
+++ b/metricbeat/docs/modules/system.asciidoc
@@ -86,7 +86,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: system
-  enabled: true
   period: 10s
   metricsets:
     - cpu
@@ -102,8 +101,8 @@ metricbeat.modules:
   process.include_top_n:
     by_cpu: 5      # include top 5 processes by CPU
     by_memory: 5   # include top 5 processes by memory
+
 - module: system
-  enabled: true
   period: 1m
   metricsets:
     - filesystem
diff --git a/metricbeat/docs/modules/vsphere.asciidoc b/metricbeat/docs/modules/vsphere.asciidoc
index e3ae92d6b8b..3ecec200c2a 100644
--- a/metricbeat/docs/modules/vsphere.asciidoc
+++ b/metricbeat/docs/modules/vsphere.asciidoc
@@ -20,7 +20,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: vsphere
   metricsets: ["datastore, host, virtualmachine"]
-  enabled: false
   period: 10s
   hosts: ["https://localhost/sdk"]
 
diff --git a/metricbeat/docs/modules/windows.asciidoc b/metricbeat/docs/modules/windows.asciidoc
index 97869af3f97..ec923ac71ed 100644
--- a/metricbeat/docs/modules/windows.asciidoc
+++ b/metricbeat/docs/modules/windows.asciidoc
@@ -19,7 +19,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: windows
   metricsets: ["perfmon"]
-  enabled: false
   period: 10s
   perfmon.counters:
 ----
diff --git a/metricbeat/docs/modules/zookeeper.asciidoc b/metricbeat/docs/modules/zookeeper.asciidoc
index 8d861ea52d2..733240437e2 100644
--- a/metricbeat/docs/modules/zookeeper.asciidoc
+++ b/metricbeat/docs/modules/zookeeper.asciidoc
@@ -25,7 +25,6 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: zookeeper
   metricsets: ["mntr"]
-  enabled: false
   period: 10s
   hosts: ["localhost:2181"]
 ----
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
index 80f06da9a66..04d5f745c09 100644
--- a/metricbeat/metricbeat.reference.yml
+++ b/metricbeat/metricbeat.reference.yml
@@ -93,7 +93,6 @@ metricbeat.modules:
 #------------------------------- Apache Module -------------------------------
 - module: apache
   metricsets: ["status"]
-  enabled: false
   period: 10s
 
   # Apache hosts
@@ -111,14 +110,12 @@ metricbeat.modules:
 #-------------------------------- ceph Module --------------------------------
 - module: ceph
   metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"]
-  enabled: false
   period: 10s
   hosts: ["localhost:5000"]
 
 #------------------------------ Couchbase Module -----------------------------
 - module: couchbase
   metricsets: ["bucket", "cluster", "node"]
-  enabled: false
   period: 10s
   hosts: ["localhost:8091"]
 
@@ -126,7 +123,6 @@ metricbeat.modules:
 - module: docker
   metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"]
   hosts: ["unix:///var/run/docker.sock"]
-  enabled: false
   period: 10s
 
   # To connect to Docker over TLS you must specify a client and CA certificate.
@@ -138,7 +134,6 @@ metricbeat.modules:
 #----------------------------- Dropwizard Module -----------------------------
 - module: dropwizard
   metricsets: ["collector"]
-  enabled: false
   period: 10s
   hosts: ["localhost:8080"]
   metrics_path: /metrics/metrics
@@ -147,16 +142,12 @@ metricbeat.modules:
 #---------------------------- elasticsearch Module ---------------------------
 - module: elasticsearch
   metricsets: ["node", "node_stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9200"]
 
-
-
 #------------------------------- golang Module -------------------------------
 - module: golang
   metricsets: ["expvar","heap"]
-  enabled: false
   period: 10s
   hosts: ["localhost:6060"]
   heap.path: "/debug/vars"
@@ -167,14 +158,12 @@ metricbeat.modules:
 #------------------------------- HAProxy Module ------------------------------
 - module: haproxy
   metricsets: ["info", "stat"]
-  enabled: false
   period: 10s
   hosts: ["tcp://127.0.0.1:14567"]
 
 #-------------------------------- HTTP Module --------------------------------
 - module: http
   metricsets: ["json"]
-  enabled: false
   period: 10s
   hosts: ["localhost:80"]
   namespace: "json_namespace"
@@ -187,7 +176,6 @@ metricbeat.modules:
 #------------------------------- Jolokia Module ------------------------------
 - module: jolokia
   metricsets: ["jmx"]
-  enabled: false
   period: 10s
   hosts: ["localhost"]
   namespace: "metrics"
@@ -199,7 +187,6 @@ metricbeat.modules:
 #-------------------------------- kafka Module -------------------------------
 - module: kafka
   metricsets: ["partition"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9092"]
 
@@ -227,14 +214,12 @@ metricbeat.modules:
 #------------------------------- kibana Module -------------------------------
 - module: kibana
   metricsets: ["status"]
-  enabled: false
   period: 10s
   hosts: ["localhost:5601"]
 
 #----------------------------- kubernetes Module -----------------------------
 # Node metrics, from kubelet:
 - module: kubernetes
-  enabled: false
   metricsets:
     - node
     - system
@@ -261,20 +246,16 @@ metricbeat.modules:
   enabled: false
   metricsets:
     - event
-  in_cluster: true
 
 #------------------------------ memcached Module -----------------------------
 - module: memcached
   metricsets: ["stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:11211"]
 
-
 #------------------------------- MongoDB Module ------------------------------
 - module: mongodb
   metricsets: ["dbstats", "status"]
-  enabled: false
   period: 10s
 
   # The hosts must be passed as MongoDB URLs in the format:
@@ -293,7 +274,6 @@ metricbeat.modules:
 #-------------------------------- MySQL Module -------------------------------
 - module: mysql
   metricsets: ["status"]
-  enabled: false
   period: 10s
 
   # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/"
@@ -325,7 +305,6 @@ metricbeat.modules:
 #------------------------------- php_fpm Module ------------------------------
 - module: php_fpm
   metricsets: ["pool"]
-  enabled: false
   period: 10s
   status_path: "/status"
   hosts: ["localhost:8080"]
@@ -342,7 +321,6 @@ metricbeat.modules:
     # Stats about every PostgreSQL process
     - activity
 
-  enabled: false
   period: 10s
 
   # The host must be passed as PostgreSQL URL. Example:
@@ -357,11 +335,9 @@ metricbeat.modules:
   # Password to use when connecting to PostgreSQL. Empty by default.
   #password: pass
 
-
 #----------------------------- Prometheus Module -----------------------------
 - module: prometheus
   metricsets: ["stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9090"]
   metrics_path: /metrics
@@ -370,7 +346,6 @@ metricbeat.modules:
 #------------------------------ rabbitmq Module ------------------------------
 - module: rabbitmq
   metricsets: ["node"]
-  enabled: false
   period: 10s
   hosts: ["localhost:15672"]
 
@@ -380,7 +355,6 @@ metricbeat.modules:
 #-------------------------------- Redis Module -------------------------------
 - module: redis
   metricsets: ["info", "keyspace"]
-  enabled: false
   period: 10s
 
   # Redis hosts
@@ -412,7 +386,6 @@ metricbeat.modules:
 #------------------------------- vsphere Module ------------------------------
 - module: vsphere
   metricsets: ["datastore, host, virtualmachine"]
-  enabled: false
   period: 10s
   hosts: ["https://localhost/sdk"]
 
@@ -424,14 +397,12 @@ metricbeat.modules:
 #------------------------------- Windows Module ------------------------------
 - module: windows
   metricsets: ["perfmon"]
-  enabled: false
   period: 10s
   perfmon.counters:
 
 #------------------------------ ZooKeeper Module -----------------------------
 - module: zookeeper
   metricsets: ["mntr"]
-  enabled: false
   period: 10s
   hosts: ["localhost:2181"]
 
diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml
index 00d3aa5500e..2be8f6cb2be 100644
--- a/metricbeat/metricbeat.yml
+++ b/metricbeat/metricbeat.yml
@@ -8,35 +8,15 @@
 # https://www.elastic.co/guide/en/beats/metricbeat/index.html
 
 #==========================  Modules configuration ============================
-metricbeat.modules:
-
-#------------------------------- System Module -------------------------------
-- module: system
-  enabled: true
-  period: 10s
-  metricsets:
-    - cpu
-    - load
-    - memory
-    - network
-    - process
-    - process_summary
-    #- core
-    #- diskio
-    #- socket
-  processes: ['.*']
-  process.include_top_n:
-    by_cpu: 5      # include top 5 processes by CPU
-    by_memory: 5   # include top 5 processes by memory
-- module: system
-  enabled: true
-  period: 1m
-  metricsets:
-    - filesystem
-    - fsstat
-  filters:
-    - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
+metricbeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
 
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  reload.period: 10s
 
 #==================== Elasticsearch template setting ==========================
 setup.template.settings:
diff --git a/metricbeat/module/apache/_meta/config.reference.yml b/metricbeat/module/apache/_meta/config.reference.yml
index 40a9ccfe14c..74f4266334f 100644
--- a/metricbeat/module/apache/_meta/config.reference.yml
+++ b/metricbeat/module/apache/_meta/config.reference.yml
@@ -1,6 +1,5 @@
 - module: apache
   metricsets: ["status"]
-  enabled: false
   period: 10s
 
   # Apache hosts
diff --git a/metricbeat/module/apache/_meta/config.yml b/metricbeat/module/apache/_meta/config.yml
index 72ea8610b7e..08d1efaf92c 100644
--- a/metricbeat/module/apache/_meta/config.yml
+++ b/metricbeat/module/apache/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: apache
   metricsets: ["status"]
-  enabled: false
   period: 10s
 
   # Apache hosts
diff --git a/metricbeat/module/ceph/_meta/config.yml b/metricbeat/module/ceph/_meta/config.yml
index c8ae607c4a0..21ee6821733 100644
--- a/metricbeat/module/ceph/_meta/config.yml
+++ b/metricbeat/module/ceph/_meta/config.yml
@@ -1,5 +1,4 @@
 - module: ceph
   metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"]
-  enabled: false
   period: 10s
   hosts: ["localhost:5000"]
diff --git a/metricbeat/module/couchbase/_meta/config.yml b/metricbeat/module/couchbase/_meta/config.yml
index dda1bb4731b..2691d77a1b4 100644
--- a/metricbeat/module/couchbase/_meta/config.yml
+++ b/metricbeat/module/couchbase/_meta/config.yml
@@ -1,5 +1,4 @@
 - module: couchbase
   metricsets: ["bucket", "cluster", "node"]
-  enabled: false
   period: 10s
   hosts: ["localhost:8091"]
diff --git a/metricbeat/module/docker/_meta/config.yml b/metricbeat/module/docker/_meta/config.yml
index 809b867afa3..87049258401 100644
--- a/metricbeat/module/docker/_meta/config.yml
+++ b/metricbeat/module/docker/_meta/config.yml
@@ -1,7 +1,6 @@
 - module: docker
   metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"]
   hosts: ["unix:///var/run/docker.sock"]
-  enabled: false
   period: 10s
 
   # To connect to Docker over TLS you must specify a client and CA certificate.
diff --git a/metricbeat/module/dropwizard/_meta/config.yml b/metricbeat/module/dropwizard/_meta/config.yml
index adf7ff28f10..6dfa3755ca0 100644
--- a/metricbeat/module/dropwizard/_meta/config.yml
+++ b/metricbeat/module/dropwizard/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: dropwizard
   metricsets: ["collector"]
-  enabled: false
   period: 10s
   hosts: ["localhost:8080"]
   metrics_path: /metrics/metrics
diff --git a/metricbeat/module/elasticsearch/_meta/config.yml b/metricbeat/module/elasticsearch/_meta/config.yml
index 443979ea549..a792d2bee9d 100644
--- a/metricbeat/module/elasticsearch/_meta/config.yml
+++ b/metricbeat/module/elasticsearch/_meta/config.yml
@@ -1,7 +1,4 @@
 - module: elasticsearch
   metricsets: ["node", "node_stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9200"]
-
-
diff --git a/metricbeat/module/golang/_meta/config.yml b/metricbeat/module/golang/_meta/config.yml
index 440b9e78055..ba9db8e1089 100644
--- a/metricbeat/module/golang/_meta/config.yml
+++ b/metricbeat/module/golang/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: golang
   metricsets: ["expvar","heap"]
-  enabled: false
   period: 10s
   hosts: ["localhost:6060"]
   heap.path: "/debug/vars"
diff --git a/metricbeat/module/haproxy/_meta/config.yml b/metricbeat/module/haproxy/_meta/config.yml
index fcee8d3b8fe..febce8d2246 100644
--- a/metricbeat/module/haproxy/_meta/config.yml
+++ b/metricbeat/module/haproxy/_meta/config.yml
@@ -1,5 +1,4 @@
 - module: haproxy
   metricsets: ["info", "stat"]
-  enabled: false
   period: 10s
   hosts: ["tcp://127.0.0.1:14567"]
diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml
index 936c435b964..aa40cfb7cc1 100644
--- a/metricbeat/module/http/_meta/config.yml
+++ b/metricbeat/module/http/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: http
   metricsets: ["json"]
-  enabled: false
   period: 10s
   hosts: ["localhost:80"]
   namespace: "json_namespace"
diff --git a/metricbeat/module/jolokia/_meta/config.yml b/metricbeat/module/jolokia/_meta/config.yml
index b7bd87fd3ad..de331606b85 100644
--- a/metricbeat/module/jolokia/_meta/config.yml
+++ b/metricbeat/module/jolokia/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: jolokia
   metricsets: ["jmx"]
-  enabled: false
   period: 10s
   hosts: ["localhost"]
   namespace: "metrics"
diff --git a/metricbeat/module/kafka/_meta/config.yml b/metricbeat/module/kafka/_meta/config.yml
index f569f71d7d8..91ce183a0f1 100644
--- a/metricbeat/module/kafka/_meta/config.yml
+++ b/metricbeat/module/kafka/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: kafka
   metricsets: ["partition"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9092"]
 
diff --git a/metricbeat/module/kibana/_meta/config.yml b/metricbeat/module/kibana/_meta/config.yml
index bd673be5658..09a30295679 100644
--- a/metricbeat/module/kibana/_meta/config.yml
+++ b/metricbeat/module/kibana/_meta/config.yml
@@ -1,5 +1,4 @@
 - module: kibana
   metricsets: ["status"]
-  enabled: false
   period: 10s
   hosts: ["localhost:5601"]
diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml
index a075952bf3a..cf00d10cd45 100644
--- a/metricbeat/module/kubernetes/_meta/config.yml
+++ b/metricbeat/module/kubernetes/_meta/config.yml
@@ -1,6 +1,5 @@
 # Node metrics, from kubelet:
 - module: kubernetes
-  enabled: false
   metricsets:
     - node
     - system
@@ -27,4 +26,3 @@
   enabled: false
   metricsets:
     - event
-  in_cluster: true
diff --git a/metricbeat/module/memcached/_meta/config.yml b/metricbeat/module/memcached/_meta/config.yml
index 85632bddb58..1b230087fab 100644
--- a/metricbeat/module/memcached/_meta/config.yml
+++ b/metricbeat/module/memcached/_meta/config.yml
@@ -1,6 +1,4 @@
 - module: memcached
   metricsets: ["stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:11211"]
-
diff --git a/metricbeat/module/mongodb/_meta/config.yml b/metricbeat/module/mongodb/_meta/config.yml
index 5eb0d599411..fb6b19e7c68 100644
--- a/metricbeat/module/mongodb/_meta/config.yml
+++ b/metricbeat/module/mongodb/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: mongodb
   metricsets: ["dbstats", "status"]
-  enabled: false
   period: 10s
 
   # The hosts must be passed as MongoDB URLs in the format:
diff --git a/metricbeat/module/mysql/_meta/config.yml b/metricbeat/module/mysql/_meta/config.yml
index 9fa5f01116d..afff5ff7201 100644
--- a/metricbeat/module/mysql/_meta/config.yml
+++ b/metricbeat/module/mysql/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: mysql
   metricsets: ["status"]
-  enabled: false
   period: 10s
 
   # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/"
diff --git a/metricbeat/module/nginx/_meta/config.yml b/metricbeat/module/nginx/_meta/config.yml
index e9bb1c84e8b..e3737ed986e 100644
--- a/metricbeat/module/nginx/_meta/config.yml
+++ b/metricbeat/module/nginx/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: nginx
   metricsets: ["stubstatus"]
-  enabled: false
   period: 10s
 
   # Nginx hosts
diff --git a/metricbeat/module/php_fpm/_meta/config.yml b/metricbeat/module/php_fpm/_meta/config.yml
index e4cdf392866..be576451404 100644
--- a/metricbeat/module/php_fpm/_meta/config.yml
+++ b/metricbeat/module/php_fpm/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: php_fpm
   metricsets: ["pool"]
-  enabled: false
   period: 10s
   status_path: "/status"
   hosts: ["localhost:8080"]
diff --git a/metricbeat/module/postgresql/_meta/config.yml b/metricbeat/module/postgresql/_meta/config.yml
index 911b029a5c3..383c66d9f4f 100644
--- a/metricbeat/module/postgresql/_meta/config.yml
+++ b/metricbeat/module/postgresql/_meta/config.yml
@@ -9,7 +9,6 @@
     # Stats about every PostgreSQL process
     - activity
 
-  enabled: false
   period: 10s
 
   # The host must be passed as PostgreSQL URL. Example:
@@ -23,4 +22,3 @@
 
   # Password to use when connecting to PostgreSQL. Empty by default.
   #password: pass
-
diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml
index 084dc2d00b8..76bee349195 100644
--- a/metricbeat/module/prometheus/_meta/config.yml
+++ b/metricbeat/module/prometheus/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: prometheus
   metricsets: ["stats"]
-  enabled: false
   period: 10s
   hosts: ["localhost:9090"]
   metrics_path: /metrics
diff --git a/metricbeat/module/rabbitmq/_meta/config.yml b/metricbeat/module/rabbitmq/_meta/config.yml
index cee569d91c2..f45427f547b 100644
--- a/metricbeat/module/rabbitmq/_meta/config.yml
+++ b/metricbeat/module/rabbitmq/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: rabbitmq
   metricsets: ["node"]
-  enabled: false
   period: 10s
   hosts: ["localhost:15672"]
 
diff --git a/metricbeat/module/redis/_meta/config.yml b/metricbeat/module/redis/_meta/config.yml
index e9d3895a40b..3f3e29ec5a4 100644
--- a/metricbeat/module/redis/_meta/config.yml
+++ b/metricbeat/module/redis/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: redis
   metricsets: ["info", "keyspace"]
-  enabled: false
   period: 10s
 
   # Redis hosts
diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml
index 111aa68a2ee..d994e02c2e7 100644
--- a/metricbeat/module/system/_meta/config.yml
+++ b/metricbeat/module/system/_meta/config.yml
@@ -1,5 +1,4 @@
 - module: system
-  enabled: true
   period: 10s
   metricsets:
     - cpu
@@ -15,8 +14,8 @@
   process.include_top_n:
     by_cpu: 5      # include top 5 processes by CPU
     by_memory: 5   # include top 5 processes by memory
+
 - module: system
-  enabled: true
   period: 1m
   metricsets:
     - filesystem
diff --git a/metricbeat/module/vsphere/_meta/config.yml b/metricbeat/module/vsphere/_meta/config.yml
index d2c73e2473d..171e1d0224b 100644
--- a/metricbeat/module/vsphere/_meta/config.yml
+++ b/metricbeat/module/vsphere/_meta/config.yml
@@ -1,6 +1,5 @@
 - module: vsphere
   metricsets: ["datastore, host, virtualmachine"]
-  enabled: false
   period: 10s
   hosts: ["https://localhost/sdk"]
 
diff --git a/metricbeat/module/windows/_meta/config.yml b/metricbeat/module/windows/_meta/config.yml
index 968ba4c5134..fc93050dabf 100644
--- a/metricbeat/module/windows/_meta/config.yml
+++ b/metricbeat/module/windows/_meta/config.yml
@@ -1,5 +1,4 @@
 - module: windows
   metricsets: ["perfmon"]
-  enabled: false
   period: 10s
   perfmon.counters:
diff --git a/metricbeat/module/zookeeper/_meta/config.yml b/metricbeat/module/zookeeper/_meta/config.yml
index 77eb186097a..63543cefdd7 100644
--- a/metricbeat/module/zookeeper/_meta/config.yml
+++ b/metricbeat/module/zookeeper/_meta/config.yml
@@ -1,5 +1,4 @@
 - module: zookeeper
   metricsets: ["mntr"]
-  enabled: false
   period: 10s
   hosts: ["localhost:2181"]
diff --git a/metricbeat/modules.d/aerospike.yml.disabled b/metricbeat/modules.d/aerospike.yml.disabled
new file mode 100644
index 00000000000..787e314dc60
--- /dev/null
+++ b/metricbeat/modules.d/aerospike.yml.disabled
@@ -0,0 +1,5 @@
+- module: aerospike
+  metricsets: ["namespace"]
+  enabled: false
+  period: 10s
+  hosts: ["localhost:3000"]
diff --git a/metricbeat/modules.d/apache.yml.disabled b/metricbeat/modules.d/apache.yml.disabled
new file mode 100644
index 00000000000..08d1efaf92c
--- /dev/null
+++ b/metricbeat/modules.d/apache.yml.disabled
@@ -0,0 +1,6 @@
+- module: apache
+  metricsets: ["status"]
+  period: 10s
+
+  # Apache hosts
+  hosts: ["http://127.0.0.1"]
diff --git a/metricbeat/modules.d/ceph.yml.disabled b/metricbeat/modules.d/ceph.yml.disabled
new file mode 100644
index 00000000000..21ee6821733
--- /dev/null
+++ b/metricbeat/modules.d/ceph.yml.disabled
@@ -0,0 +1,4 @@
+- module: ceph
+  metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"]
+  period: 10s
+  hosts: ["localhost:5000"]
diff --git a/metricbeat/modules.d/couchbase.yml.disabled b/metricbeat/modules.d/couchbase.yml.disabled
new file mode 100644
index 00000000000..2691d77a1b4
--- /dev/null
+++ b/metricbeat/modules.d/couchbase.yml.disabled
@@ -0,0 +1,4 @@
+- module: couchbase
+  metricsets: ["bucket", "cluster", "node"]
+  period: 10s
+  hosts: ["localhost:8091"]
diff --git a/metricbeat/modules.d/docker.yml.disabled b/metricbeat/modules.d/docker.yml.disabled
new file mode 100644
index 00000000000..87049258401
--- /dev/null
+++ b/metricbeat/modules.d/docker.yml.disabled
@@ -0,0 +1,10 @@
+- module: docker
+  metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"]
+  hosts: ["unix:///var/run/docker.sock"]
+  period: 10s
+
+  # To connect to Docker over TLS you must specify a client and CA certificate.
+  #ssl:
+    #certificate_authority: "/etc/pki/root/ca.pem"
+    #certificate:           "/etc/pki/client/cert.pem"
+    #key:                   "/etc/pki/client/cert.key"
diff --git a/metricbeat/modules.d/dropwizard.yml.disabled b/metricbeat/modules.d/dropwizard.yml.disabled
new file mode 100644
index 00000000000..6dfa3755ca0
--- /dev/null
+++ b/metricbeat/modules.d/dropwizard.yml.disabled
@@ -0,0 +1,6 @@
+- module: dropwizard
+  metricsets: ["collector"]
+  period: 10s
+  hosts: ["localhost:8080"]
+  metrics_path: /metrics/metrics
+  namespace: example
diff --git a/metricbeat/modules.d/elasticsearch.yml.disabled b/metricbeat/modules.d/elasticsearch.yml.disabled
new file mode 100644
index 00000000000..a792d2bee9d
--- /dev/null
+++ b/metricbeat/modules.d/elasticsearch.yml.disabled
@@ -0,0 +1,4 @@
+- module: elasticsearch
+  metricsets: ["node", "node_stats"]
+  period: 10s
+  hosts: ["localhost:9200"]
diff --git a/metricbeat/modules.d/golang.yml.disabled b/metricbeat/modules.d/golang.yml.disabled
new file mode 100644
index 00000000000..ba9db8e1089
--- /dev/null
+++ b/metricbeat/modules.d/golang.yml.disabled
@@ -0,0 +1,8 @@
+- module: golang
+  metricsets: ["expvar","heap"]
+  period: 10s
+  hosts: ["localhost:6060"]
+  heap.path: "/debug/vars"
+  expvar:
+    namespace: "example"
+    path: "/debug/vars"
diff --git a/metricbeat/modules.d/haproxy.yml.disabled b/metricbeat/modules.d/haproxy.yml.disabled
new file mode 100644
index 00000000000..febce8d2246
--- /dev/null
+++ b/metricbeat/modules.d/haproxy.yml.disabled
@@ -0,0 +1,4 @@
+- module: haproxy
+  metricsets: ["info", "stat"]
+  period: 10s
+  hosts: ["tcp://127.0.0.1:14567"]
diff --git a/metricbeat/modules.d/http.yml.disabled b/metricbeat/modules.d/http.yml.disabled
new file mode 100644
index 00000000000..aa40cfb7cc1
--- /dev/null
+++ b/metricbeat/modules.d/http.yml.disabled
@@ -0,0 +1,10 @@
+- module: http
+  metricsets: ["json"]
+  period: 10s
+  hosts: ["localhost:80"]
+  namespace: "json_namespace"
+  path: "/"
+  #body: ""
+  #method: "GET"
+  #request.enabled: false
+  #response.enabled: false
diff --git a/metricbeat/modules.d/jolokia.yml.disabled b/metricbeat/modules.d/jolokia.yml.disabled
new file mode 100644
index 00000000000..de331606b85
--- /dev/null
+++ b/metricbeat/modules.d/jolokia.yml.disabled
@@ -0,0 +1,9 @@
+- module: jolokia
+  metricsets: ["jmx"]
+  period: 10s
+  hosts: ["localhost"]
+  namespace: "metrics"
+  path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
+  jmx.mapping:
+  jmx.application:
+  jmx.instance:
diff --git a/metricbeat/modules.d/kafka.yml.disabled b/metricbeat/modules.d/kafka.yml.disabled
new file mode 100644
index 00000000000..91ce183a0f1
--- /dev/null
+++ b/metricbeat/modules.d/kafka.yml.disabled
@@ -0,0 +1,25 @@
+- module: kafka
+  metricsets: ["partition"]
+  period: 10s
+  hosts: ["localhost:9092"]
+
+  #client_id: metricbeat
+  #retries: 3
+  #backoff: 250ms
+
+  # List of Topics to query metadata for. If empty, all topics will be queried.
+  #topics: []
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+  # SASL authentication
+  #username: ""
+  #password: ""
diff --git a/metricbeat/modules.d/kibana.yml.disabled b/metricbeat/modules.d/kibana.yml.disabled
new file mode 100644
index 00000000000..09a30295679
--- /dev/null
+++ b/metricbeat/modules.d/kibana.yml.disabled
@@ -0,0 +1,4 @@
+- module: kibana
+  metricsets: ["status"]
+  period: 10s
+  hosts: ["localhost:5601"]
diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled
new file mode 100644
index 00000000000..cf00d10cd45
--- /dev/null
+++ b/metricbeat/modules.d/kubernetes.yml.disabled
@@ -0,0 +1,28 @@
+# Node metrics, from kubelet:
+- module: kubernetes
+  metricsets:
+    - node
+    - system
+    - pod
+    - container
+    - volume
+  period: 10s
+  hosts: ["localhost:10255"]
+
+# State metrics from kube-state-metrics service:
+- module: kubernetes
+  enabled: false
+  metricsets:
+    - state_node
+    - state_deployment
+    - state_replicaset
+    - state_pod
+    - state_container
+  period: 10s
+  hosts: ["kube-state-metrics:8080"]
+
+# Kubernetes events
+- module: kubernetes
+  enabled: false
+  metricsets:
+    - event
diff --git a/metricbeat/modules.d/memcached.yml.disabled b/metricbeat/modules.d/memcached.yml.disabled
new file mode 100644
index 00000000000..1b230087fab
--- /dev/null
+++ b/metricbeat/modules.d/memcached.yml.disabled
@@ -0,0 +1,4 @@
+- module: memcached
+  metricsets: ["stats"]
+  period: 10s
+  hosts: ["localhost:11211"]
diff --git a/metricbeat/modules.d/mysql.yml.disabled b/metricbeat/modules.d/mysql.yml.disabled
new file mode 100644
index 00000000000..afff5ff7201
--- /dev/null
+++ b/metricbeat/modules.d/mysql.yml.disabled
@@ -0,0 +1,17 @@
+- module: mysql
+  metricsets: ["status"]
+  period: 10s
+
+  # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/"
+  # The username and password can either be set in the DSN or using the username
+  # and password config options. Those specified in the DSN take precedence.
+  hosts: ["root:secret@tcp(127.0.0.1:3306)/"]
+
+  # Username of hosts. Empty by default.
+  #username: root
+
+  # Password of hosts. Empty by default.
+  #password: secret
+
+  # By setting raw to true, all raw fields from the status metricset will be added to the event.
+  #raw: false
diff --git a/metricbeat/modules.d/nginx.yml.disabled b/metricbeat/modules.d/nginx.yml.disabled
new file mode 100644
index 00000000000..e3737ed986e
--- /dev/null
+++ b/metricbeat/modules.d/nginx.yml.disabled
@@ -0,0 +1,9 @@
+- module: nginx
+  metricsets: ["stubstatus"]
+  period: 10s
+
+  # Nginx hosts
+  hosts: ["http://127.0.0.1"]
+
+  # Path to server status. Default server-status
+  #server_status_path: "server-status"
diff --git a/metricbeat/modules.d/php_fpm.yml.disabled b/metricbeat/modules.d/php_fpm.yml.disabled
new file mode 100644
index 00000000000..be576451404
--- /dev/null
+++ b/metricbeat/modules.d/php_fpm.yml.disabled
@@ -0,0 +1,5 @@
+- module: php_fpm
+  metricsets: ["pool"]
+  period: 10s
+  status_path: "/status"
+  hosts: ["localhost:8080"]
diff --git a/metricbeat/modules.d/postgresql.yml.disabled b/metricbeat/modules.d/postgresql.yml.disabled
new file mode 100644
index 00000000000..383c66d9f4f
--- /dev/null
+++ b/metricbeat/modules.d/postgresql.yml.disabled
@@ -0,0 +1,24 @@
+- module: postgresql
+  metricsets:
+    # Stats about every PostgreSQL database
+    - database
+
+    # Stats about the background writer process's activity
+    - bgwriter
+
+    # Stats about every PostgreSQL process
+    - activity
+
+  period: 10s
+
+  # The host must be passed as PostgreSQL URL. Example:
+  # postgres://localhost:5432?sslmode=disable
+  # The available parameters are documented here:
+  # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
+  hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled
new file mode 100644
index 00000000000..76bee349195
--- /dev/null
+++ b/metricbeat/modules.d/prometheus.yml.disabled
@@ -0,0 +1,6 @@
+- module: prometheus
+  metricsets: ["stats"]
+  period: 10s
+  hosts: ["localhost:9090"]
+  metrics_path: /metrics
+  #namespace: example
diff --git a/metricbeat/modules.d/rabbitmq.yml.disabled b/metricbeat/modules.d/rabbitmq.yml.disabled
new file mode 100644
index 00000000000..f45427f547b
--- /dev/null
+++ b/metricbeat/modules.d/rabbitmq.yml.disabled
@@ -0,0 +1,7 @@
+- module: rabbitmq
+  metricsets: ["node"]
+  period: 10s
+  hosts: ["localhost:15672"]
+
+  username: guest
+  password: guest
diff --git a/metricbeat/modules.d/redis.yml.disabled b/metricbeat/modules.d/redis.yml.disabled
new file mode 100644
index 00000000000..3f3e29ec5a4
--- /dev/null
+++ b/metricbeat/modules.d/redis.yml.disabled
@@ -0,0 +1,29 @@
+- module: redis
+  metricsets: ["info", "keyspace"]
+  period: 10s
+
+  # Redis hosts
+  hosts: ["127.0.0.1:6379"]
+
+  # Timeout after which time a metricset should return an error
+  # Timeout is by default defined as period, as a fetch of a metricset
+  # should never take longer then period, as otherwise calls can pile up.
+  #timeout: 1s
+
+  # Optional fields to be added to each event
+  #fields:
+  #  datacenter: west
+
+  # Network type to be used for redis connection. Default: tcp
+  #network: tcp
+
+  # Max number of concurrent connections. Default: 10
+  #maxconn: 10
+
+  # Filters can be used to reduce the number of fields sent.
+  #filters:
+  #  - include_fields:
+  #      fields: ["stats"]
+
+  # Redis AUTH password. Empty by default.
+  #password: foobared
diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml
new file mode 100644
index 00000000000..d994e02c2e7
--- /dev/null
+++ b/metricbeat/modules.d/system.yml
@@ -0,0 +1,24 @@
+- module: system
+  period: 10s
+  metricsets:
+    - cpu
+    - load
+    - memory
+    - network
+    - process
+    - process_summary
+    #- core
+    #- diskio
+    #- socket
+  processes: ['.*']
+  process.include_top_n:
+    by_cpu: 5      # include top 5 processes by CPU
+    by_memory: 5   # include top 5 processes by memory
+
+- module: system
+  period: 1m
+  metricsets:
+    - filesystem
+    - fsstat
+  filters:
+    - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
diff --git a/metricbeat/modules.d/vsphere.yml.disabled b/metricbeat/modules.d/vsphere.yml.disabled
new file mode 100644
index 00000000000..171e1d0224b
--- /dev/null
+++ b/metricbeat/modules.d/vsphere.yml.disabled
@@ -0,0 +1,9 @@
+- module: vsphere
+  metricsets: ["datastore, host, virtualmachine"]
+  period: 10s
+  hosts: ["https://localhost/sdk"]
+
+  username: "user"
+  password: "password"
+  # If insecure is true, don't verify the server's certificate chain
+  insecure: false
diff --git a/metricbeat/modules.d/windows.yml.disabled b/metricbeat/modules.d/windows.yml.disabled
new file mode 100644
index 00000000000..fc93050dabf
--- /dev/null
+++ b/metricbeat/modules.d/windows.yml.disabled
@@ -0,0 +1,4 @@
+- module: windows
+  metricsets: ["perfmon"]
+  period: 10s
+  perfmon.counters:
diff --git a/metricbeat/modules.d/zookeeper.yml.disabled b/metricbeat/modules.d/zookeeper.yml.disabled
new file mode 100644
index 00000000000..63543cefdd7
--- /dev/null
+++ b/metricbeat/modules.d/zookeeper.yml.disabled
@@ -0,0 +1,4 @@
+- module: zookeeper
+  metricsets: ["mntr"]
+  period: 10s
+  hosts: ["localhost:2181"]

From 4298eb18469b5cf5931384f48e73d58995a19b46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Tue, 20 Jun 2017 17:15:56 +0200
Subject: [PATCH 04/14] Enable `modules` command for metricbeat

---
 libbeat/beat/beat.go      |  6 ++---
 libbeat/cmd/modules.go    | 49 +++++++++++++++++++++++++++++++++------
 metricbeat/cmd/modules.go | 33 ++++++++++++++++++++++++++
 metricbeat/cmd/root.go    |  1 +
 4 files changed, 79 insertions(+), 10 deletions(-)
 create mode 100644 metricbeat/cmd/modules.go

diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go
index 1e6387ba2e1..ec4e77fdf13 100644
--- a/libbeat/beat/beat.go
+++ b/libbeat/beat/beat.go
@@ -213,8 +213,8 @@ func (b *Beat) Init() error {
 	return nil
 }
 
-// load the beats config section
-func (b *Beat) config() (*common.Config, error) {
+// BeatConfig returns config section for this beat
+func (b *Beat) BeatConfig() (*common.Config, error) {
 	configName := strings.ToLower(b.Info.Beat)
 	if b.RawConfig.HasField(configName) {
 		sub, err := b.RawConfig.Child(configName, -1)
@@ -231,7 +231,7 @@ func (b *Beat) config() (*common.Config, error) {
 // create and return the beater, this method also initializes all needed items,
 // including template registering, publisher, xpack monitoring
 func (b *Beat) createBeater(bt Creator) (Beater, error) {
-	sub, err := b.config()
+	sub, err := b.BeatConfig()
 	if err != nil {
 		return nil, err
 	}
diff --git a/libbeat/cmd/modules.go b/libbeat/cmd/modules.go
index a660e9973d8..c75c10954aa 100644
--- a/libbeat/cmd/modules.go
+++ b/libbeat/cmd/modules.go
@@ -5,8 +5,12 @@ import (
 	"os"
 
 	"github.com/spf13/cobra"
+
+	"github.com/elastic/beats/libbeat/beat"
 )
 
+// ModulesManager interface provides all actions needed to implement modules command
+// (to list, enable & disable modules)
 type ModulesManager interface {
 	ListEnabled() []string
 	ListDisabled() []string
@@ -16,26 +20,53 @@ type ModulesManager interface {
 	Disable(name string) error
 }
 
+// ModulesManagerFactory builds and return a ModulesManager for the given Beat
+type ModulesManagerFactory func(beat *beat.Beat) (ModulesManager, error)
+
 // GenModulesCmd initializes a command to manage a modules.d folder, it offers
 // list, enable and siable actions
-func GenModulesCmd(name, version string, modules ModulesManager) *cobra.Command {
+func GenModulesCmd(name, version string, modulesFactory ModulesManagerFactory) *cobra.Command {
 	modulesCmd := cobra.Command{
 		Use:   "modules",
 		Short: "Manage configured modules",
 	}
 
-	modulesCmd.AddCommand(genListModulesCmd(modules))
-	modulesCmd.AddCommand(genEnableModulesCmd(modules))
-	modulesCmd.AddCommand(genDisableModulesCmd(modules))
+	modulesCmd.AddCommand(genListModulesCmd(name, version, modulesFactory))
+	modulesCmd.AddCommand(genEnableModulesCmd(name, version, modulesFactory))
+	modulesCmd.AddCommand(genDisableModulesCmd(name, version, modulesFactory))
 
 	return &modulesCmd
 }
 
-func genListModulesCmd(modules ModulesManager) *cobra.Command {
+// Instantiate a modules manager or die trying
+func getModules(name, version string, modulesFactory ModulesManagerFactory) ModulesManager {
+	b, err := beat.New(name, version)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
+		os.Exit(1)
+	}
+
+	if err = b.Init(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err)
+		os.Exit(1)
+	}
+
+	manager, err := modulesFactory(b)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error in modules manager: %s\n", err)
+		os.Exit(1)
+	}
+
+	return manager
+}
+
+func genListModulesCmd(name, version string, modulesFactory ModulesManagerFactory) *cobra.Command {
 	return &cobra.Command{
 		Use:   "list",
 		Short: "List modules",
 		Run: func(cmd *cobra.Command, args []string) {
+			modules := getModules(name, version, modulesFactory)
+
 			fmt.Println("Enabled:")
 			for _, module := range modules.ListEnabled() {
 				fmt.Println(module)
@@ -49,11 +80,13 @@ func genListModulesCmd(modules ModulesManager) *cobra.Command {
 	}
 }
 
-func genEnableModulesCmd(modules ModulesManager) *cobra.Command {
+func genEnableModulesCmd(name, version string, modulesFactory ModulesManagerFactory) *cobra.Command {
 	return &cobra.Command{
 		Use:   "enable MODULE...",
 		Short: "Enable one or more given modules",
 		Run: func(cmd *cobra.Command, args []string) {
+			modules := getModules(name, version, modulesFactory)
+
 			for _, module := range args {
 				if !modules.Exists(module) {
 					fmt.Printf("Module %s doesn't exists!\n", module)
@@ -76,11 +109,13 @@ func genEnableModulesCmd(modules ModulesManager) *cobra.Command {
 	}
 }
 
-func genDisableModulesCmd(modules ModulesManager) *cobra.Command {
+func genDisableModulesCmd(name, version string, modulesFactory ModulesManagerFactory) *cobra.Command {
 	return &cobra.Command{
 		Use:   "disable MODULE...",
 		Short: "Disable one or more given modules",
 		Run: func(cmd *cobra.Command, args []string) {
+			modules := getModules(name, version, modulesFactory)
+
 			for _, module := range args {
 				if !modules.Exists(module) {
 					fmt.Fprintf(os.Stderr, "Module %s doesn't exists!\n", module)
diff --git a/metricbeat/cmd/modules.go b/metricbeat/cmd/modules.go
new file mode 100644
index 00000000000..0df5a50ce6a
--- /dev/null
+++ b/metricbeat/cmd/modules.go
@@ -0,0 +1,33 @@
+package cmd
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/cfgfile"
+	"github.com/elastic/beats/libbeat/cmd"
+)
+
+func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
+	config, err := beat.BeatConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "initialization error")
+	}
+
+	glob, err := config.String("config.modules.path", -1)
+	if err != nil {
+		return nil, errors.Errorf("modules management requires 'metricbeat.config.modules.path' setting")
+	}
+
+	if !strings.HasSuffix(glob, ".yml") {
+		return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob)
+	}
+
+	modulesManager, err := cfgfile.NewGlobManager(glob, ".yml", ".disabled")
+	if err != nil {
+		return nil, errors.Wrap(err, "initialization error")
+	}
+	return modulesManager, nil
+}
diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go
index bb01846fb1e..0d81f744bc7 100644
--- a/metricbeat/cmd/root.go
+++ b/metricbeat/cmd/root.go
@@ -23,4 +23,5 @@ func init() {
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs"))
 
 	RootCmd = cmd.GenRootCmdWithRunFlags(Name, "", beater.New, runFlags)
+	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
 }

From 358e016c6fee52e8fcd95ffc0b54b367d8160949 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Mon, 26 Jun 2017 21:14:07 +0200
Subject: [PATCH 05/14] Fix windows tests

---
 libbeat/cfgfile/glob_manager_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libbeat/cfgfile/glob_manager_test.go b/libbeat/cfgfile/glob_manager_test.go
index 8d97ad89998..c651986d14f 100644
--- a/libbeat/cfgfile/glob_manager_test.go
+++ b/libbeat/cfgfile/glob_manager_test.go
@@ -79,8 +79,8 @@ func TestGlobManager(t *testing.T) {
 	}
 
 	assert.Equal(t, files, []string{
-		dir + "/config1.yml",
-		dir + "/config2.yml.disabled",
-		dir + "/config3.yml",
+		filepath.Join(dir, "config1.yml"),
+		filepath.Join(dir, "config2.yml.disabled"),
+		filepath.Join(dir, "config3.yml"),
 	})
 }

From be394f248fc01161ec9fdf73142736f1c6c9013e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Wed, 28 Jun 2017 15:38:24 +0200
Subject: [PATCH 06/14] Add filebeat modules.d support

---
 filebeat/Makefile                       |  8 +-
 filebeat/_meta/common.p2.yml            | 12 +++
 filebeat/cmd/modules.go                 | 33 +++++++++
 filebeat/cmd/root.go                    |  1 +
 filebeat/filebeat.yml                   | 99 +++----------------------
 filebeat/modules.d/apache2.yml.disabled | 16 ++++
 filebeat/modules.d/auditd.yml.disabled  |  8 ++
 filebeat/modules.d/icinga.yml.disabled  | 24 ++++++
 filebeat/modules.d/mysql.yml.disabled   | 16 ++++
 filebeat/modules.d/nginx.yml.disabled   | 16 ++++
 filebeat/modules.d/system.yml.disabled  | 16 ++++
 11 files changed, 159 insertions(+), 90 deletions(-)
 create mode 100644 filebeat/cmd/modules.go
 create mode 100644 filebeat/modules.d/apache2.yml.disabled
 create mode 100644 filebeat/modules.d/auditd.yml.disabled
 create mode 100644 filebeat/modules.d/icinga.yml.disabled
 create mode 100644 filebeat/modules.d/mysql.yml.disabled
 create mode 100644 filebeat/modules.d/nginx.yml.disabled
 create mode 100644 filebeat/modules.d/system.yml.disabled

diff --git a/filebeat/Makefile b/filebeat/Makefile
index 9a3bf4b44d6..7b14fa28200 100644
--- a/filebeat/Makefile
+++ b/filebeat/Makefile
@@ -36,11 +36,13 @@ modules:
 .PHONY: configs
 configs: python-env
 	@cp ${ES_BEATS}/filebeat/_meta/common.p1.yml _meta/beat.yml
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml
 	@cat ${ES_BEATS}/filebeat/_meta/common.p2.yml >> _meta/beat.yml
-	@cp ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml _meta/beat.reference.yml
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
+	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml > _meta/beat.reference.yml
+	@. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
 	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p2.yml >> _meta/beat.reference.yml
+	@rm -rf modules.d && mkdir -p modules.d
+	@for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
+	@chmod go-w modules.d/*
 
 # Collects all module docs
 .PHONY: collect-docs
diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/common.p2.yml
index ca2306d617b..0a760c280a0 100644
--- a/filebeat/_meta/common.p2.yml
+++ b/filebeat/_meta/common.p2.yml
@@ -53,3 +53,15 @@ filebeat.prospectors:
   # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
   #multiline.match: after
 
+
+#============================= Filebeat modules ===============================
+
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  reload.period: 10s
diff --git a/filebeat/cmd/modules.go b/filebeat/cmd/modules.go
new file mode 100644
index 00000000000..4bf888266f4
--- /dev/null
+++ b/filebeat/cmd/modules.go
@@ -0,0 +1,33 @@
+package cmd
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/cfgfile"
+	"github.com/elastic/beats/libbeat/cmd"
+)
+
+func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
+	config, err := beat.BeatConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "initialization error")
+	}
+
+	glob, err := config.String("config.modules.path", -1)
+	if err != nil {
+		return nil, errors.Errorf("modules management requires 'filebeat.config.modules.path' setting")
+	}
+
+	if !strings.HasSuffix(glob, ".yml") {
+		return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob)
+	}
+
+	modulesManager, err := cfgfile.NewGlobManager(glob, ".yml", ".disabled")
+	if err != nil {
+		return nil, errors.Wrap(err, "initialization error")
+	}
+	return modulesManager, nil
+}
diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go
index 4b685b657ef..6d44116b25e 100644
--- a/filebeat/cmd/root.go
+++ b/filebeat/cmd/root.go
@@ -25,4 +25,5 @@ func init() {
 	RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M"))
 	RootCmd.ConfigTestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
 	RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
+	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
 }
diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
index d941334e647..ab76fd48715 100644
--- a/filebeat/filebeat.yml
+++ b/filebeat/filebeat.yml
@@ -7,93 +7,6 @@
 # You can find the full configuration reference here:
 # https://www.elastic.co/guide/en/beats/filebeat/index.html
 
-
-#==========================  Modules configuration ============================
-filebeat.modules:
-
-#------------------------------- System Module -------------------------------
-#- module: system
-  # Syslog
-  #syslog:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-  # Authorization logs
-  #auth:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-#------------------------------- Apache2 Module ------------------------------
-#- module: apache2
-  # Access logs
-  #access:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-  # Error logs
-  #error:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-#------------------------------- Auditd Module -------------------------------
-#- module: auditd
-  #log:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-
-#-------------------------------- MySQL Module -------------------------------
-#- module: mysql
-  # Error logs
-  #error:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-  # Slow logs
-  #slowlog:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-#-------------------------------- Nginx Module -------------------------------
-#- module: nginx
-  # Access logs
-  #access:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-  # Error logs
-  #error:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-
 # For more available modules and options, please see the filebeat.reference.yml sample
 # configuration file.
 
@@ -150,6 +63,18 @@ filebeat.prospectors:
   #multiline.match: after
 
 
+#============================= Filebeat modules ===============================
+
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  reload.period: 10s
+
 #================================ General =====================================
 
 # The name of the shipper that publishes the network data. It can be used to group
diff --git a/filebeat/modules.d/apache2.yml.disabled b/filebeat/modules.d/apache2.yml.disabled
new file mode 100644
index 00000000000..6ccb548ad4f
--- /dev/null
+++ b/filebeat/modules.d/apache2.yml.disabled
@@ -0,0 +1,16 @@
+#- module: apache2
+  # Access logs
+  #access:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
diff --git a/filebeat/modules.d/auditd.yml.disabled b/filebeat/modules.d/auditd.yml.disabled
new file mode 100644
index 00000000000..5669924ba0b
--- /dev/null
+++ b/filebeat/modules.d/auditd.yml.disabled
@@ -0,0 +1,8 @@
+#- module: auditd
+  #log:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
diff --git a/filebeat/modules.d/icinga.yml.disabled b/filebeat/modules.d/icinga.yml.disabled
new file mode 100644
index 00000000000..7657f4467ee
--- /dev/null
+++ b/filebeat/modules.d/icinga.yml.disabled
@@ -0,0 +1,24 @@
+#- module: icinga
+  # Main logs
+  #main:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Debug logs
+  #debug:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Startup logs
+  #startup:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
diff --git a/filebeat/modules.d/mysql.yml.disabled b/filebeat/modules.d/mysql.yml.disabled
new file mode 100644
index 00000000000..aadb5631092
--- /dev/null
+++ b/filebeat/modules.d/mysql.yml.disabled
@@ -0,0 +1,16 @@
+#- module: mysql
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Slow logs
+  #slowlog:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
diff --git a/filebeat/modules.d/nginx.yml.disabled b/filebeat/modules.d/nginx.yml.disabled
new file mode 100644
index 00000000000..e1b45939726
--- /dev/null
+++ b/filebeat/modules.d/nginx.yml.disabled
@@ -0,0 +1,16 @@
+#- module: nginx
+  # Access logs
+  #access:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Error logs
+  #error:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled
new file mode 100644
index 00000000000..d31ca8bc6ea
--- /dev/null
+++ b/filebeat/modules.d/system.yml.disabled
@@ -0,0 +1,16 @@
+#- module: system
+  # Syslog
+  #syslog:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:
+
+  # Authorization logs
+  #auth:
+    #enabled: true
+
+    # Set custom paths for the log files. If left empty,
+    # Filebeat will choose the paths depending on your OS.
+    #var.paths:

From a0b790f08d89724a61c2d8196ff072f6f207eb03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Wed, 28 Jun 2017 18:11:41 +0200
Subject: [PATCH 07/14] Do not fail if system.yml is not present

---
 metricbeat/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metricbeat/Makefile b/metricbeat/Makefile
index ae419456b31..9b2eabe5967 100644
--- a/metricbeat/Makefile
+++ b/metricbeat/Makefile
@@ -54,7 +54,7 @@ configs: python-env
 	@for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
 	@chmod go-w modules.d/*
 	# Enable system by default:
-	@mv modules.d/system.yml.disabled modules.d/system.yml
+	@if [ -f modules.d/system.yml.disabled ]; then mv modules.d/system.yml.disabled modules.d/system.yml; fi
 
 # Generates imports for all modules and metricsets
 .PHONY: imports

From 5d42dbd2f919a0fd57ff9e373b3c46664fe2d581 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Thu, 29 Jun 2017 16:48:09 +0200
Subject: [PATCH 08/14] Comment out `reload.period` settings to make clear it's
 disabled

---
 filebeat/_meta/common.p2.yml | 2 +-
 filebeat/filebeat.yml        | 2 +-
 metricbeat/_meta/setup.yml   | 2 +-
 metricbeat/metricbeat.yml    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/common.p2.yml
index 0a760c280a0..873910954ee 100644
--- a/filebeat/_meta/common.p2.yml
+++ b/filebeat/_meta/common.p2.yml
@@ -64,4 +64,4 @@ filebeat.config.modules:
   reload.enabled: false
 
   # Period on which files under path should be checked for changes
-  reload.period: 10s
+  #reload.period: 10s
diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
index ab76fd48715..a49d550f818 100644
--- a/filebeat/filebeat.yml
+++ b/filebeat/filebeat.yml
@@ -73,7 +73,7 @@ filebeat.config.modules:
   reload.enabled: false
 
   # Period on which files under path should be checked for changes
-  reload.period: 10s
+  #reload.period: 10s
 
 #================================ General =====================================
 
diff --git a/metricbeat/_meta/setup.yml b/metricbeat/_meta/setup.yml
index 611f841e45f..0f833162283 100644
--- a/metricbeat/_meta/setup.yml
+++ b/metricbeat/_meta/setup.yml
@@ -8,7 +8,7 @@ metricbeat.config.modules:
   reload.enabled: false
 
   # Period on which files under path should be checked for changes
-  reload.period: 10s
+  #reload.period: 10s
 
 #==================== Elasticsearch template setting ==========================
 setup.template.settings:
diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml
index 2be8f6cb2be..63243270998 100644
--- a/metricbeat/metricbeat.yml
+++ b/metricbeat/metricbeat.yml
@@ -16,7 +16,7 @@ metricbeat.config.modules:
   reload.enabled: false
 
   # Period on which files under path should be checked for changes
-  reload.period: 10s
+  #reload.period: 10s
 
 #==================== Elasticsearch template setting ==========================
 setup.template.settings:

From 9638526819151c828abf9bdd264b4ccd199249b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Thu, 29 Jun 2017 19:11:24 +0200
Subject: [PATCH 09/14] Add system tests

---
 filebeat/tests/system/test_cmd.py             | 115 ++++++++++++++++++
 .../tests/system/config/metricbeat.yml.j2     |   2 +-
 metricbeat/tests/system/test_cmd.py           | 111 +++++++++++++++++
 3 files changed, 227 insertions(+), 1 deletion(-)
 create mode 100644 filebeat/tests/system/test_cmd.py
 create mode 100644 metricbeat/tests/system/test_cmd.py

diff --git a/filebeat/tests/system/test_cmd.py b/filebeat/tests/system/test_cmd.py
new file mode 100644
index 00000000000..94540b52d78
--- /dev/null
+++ b/filebeat/tests/system/test_cmd.py
@@ -0,0 +1,115 @@
+import os
+import filebeat
+
+
+class TestCommands(filebeat.BaseTest):
+    """
+    Test filebeat subcommands
+    """
+
+    def setUp(self):
+        super(TestCommands, self).setUp()
+
+        # Enable modules reload with default paths
+        self.render_config_template(
+            reload=True,
+            reload_type="modules",
+            reload_path="${path.config}/modules.d/*.yml",
+        )
+        os.mkdir(self.working_dir + "/modules.d")
+
+    def test_modules_list(self):
+        """
+        Test modules list command
+        """
+        self.touch(self.working_dir + "/modules.d/enabled.yml")
+        self.touch(self.working_dir + "/modules.d/disabled.yml.disabled")
+
+        exit_code = self.run_beat(extra_args=["modules", "list"])
+
+        assert exit_code == 0
+        assert "Enabled:\nenabled" in self.get_log()
+        assert "Disabled:\ndisabled" in self.get_log()
+
+        # Add one more disabled module
+        self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled")
+        exit_code = self.run_beat(extra_args=["modules", "list"])
+
+        assert exit_code == 0
+        assert "Enabled:\nenabled" in self.get_log()
+        assert "Disabled:\ndisabled\ndisabled2" in self.get_log()
+
+    def test_modules_enable(self):
+        """
+        Test modules enable command
+        """
+        self.touch(self.working_dir + "/modules.d/enabled.yml")
+        self.touch(self.working_dir + "/modules.d/disabled1.yml.disabled")
+        self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled")
+        self.touch(self.working_dir + "/modules.d/disabled3.yml.disabled")
+
+        # Enable one module
+        exit_code = self.run_beat(
+            extra_args=["modules", "enable", "disabled1"])
+        assert exit_code == 0
+
+        assert self.log_contains("Enabled disabled1")
+        assert os.path.exists(self.working_dir + "/modules.d/disabled1.yml")
+        assert not os.path.exists(
+            self.working_dir + "/modules.d/disabled1.yml.disabled")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/disabled2.yml.disabled")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/disabled3.yml.disabled")
+
+        # Enable several modules at once:
+        exit_code = self.run_beat(
+            extra_args=["modules", "enable", "disabled2", "disabled3"])
+        assert exit_code == 0
+
+        assert self.log_contains("Enabled disabled2")
+        assert self.log_contains("Enabled disabled3")
+        assert os.path.exists(self.working_dir + "/modules.d/disabled2.yml")
+        assert os.path.exists(self.working_dir + "/modules.d/disabled3.yml")
+        assert not os.path.exists(
+            self.working_dir + "/modules.d/disabled2.yml.disabled")
+        assert not os.path.exists(
+            self.working_dir + "/modules.d/disabled3.yml.disabled")
+
+    def test_modules_disable(self):
+        """
+        Test modules disable command
+        """
+        self.touch(self.working_dir + "/modules.d/enabled1.yml")
+        self.touch(self.working_dir + "/modules.d/enabled2.yml")
+        self.touch(self.working_dir + "/modules.d/enabled3.yml")
+        self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled")
+
+        # Disable one module
+        exit_code = self.run_beat(
+            extra_args=["modules", "disable", "enabled1"])
+        assert exit_code == 0
+
+        assert self.log_contains("Disabled enabled1")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/enabled1.yml.disabled")
+        assert not os.path.exists(self.working_dir + "/modules.d/enabled1.yml")
+        assert os.path.exists(self.working_dir + "/modules.d/enabled2.yml")
+        assert os.path.exists(self.working_dir + "/modules.d/enabled3.yml")
+
+        # Disable several modules at once:
+        exit_code = self.run_beat(
+            extra_args=["modules", "disable", "enabled2", "enabled3"])
+        assert exit_code == 0
+
+        assert self.log_contains("Disabled enabled2")
+        assert self.log_contains("Disabled enabled3")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/enabled2.yml.disabled")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/enabled3.yml.disabled")
+        assert not os.path.exists(self.working_dir + "/modules.d/enabled2.yml")
+        assert not os.path.exists(self.working_dir + "/modules.d/enabled3.yml")
+
+    def touch(self, path):
+        open(path, 'a').close()
diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2
index d9ff62b1943..4ca002b67e2 100644
--- a/metricbeat/tests/system/config/metricbeat.yml.j2
+++ b/metricbeat/tests/system/config/metricbeat.yml.j2
@@ -93,7 +93,7 @@ metricbeat.modules:
 
 {% if reload -%}
 metricbeat.config.modules:
-  path: {{ reload_path }}
+  path: {{ reload_path|default("${path.config}/modules.d/*.yml") }}
   reload.period: 1s
   reload.enabled: true
 {% endif -%}
diff --git a/metricbeat/tests/system/test_cmd.py b/metricbeat/tests/system/test_cmd.py
new file mode 100644
index 00000000000..a3657b42d70
--- /dev/null
+++ b/metricbeat/tests/system/test_cmd.py
@@ -0,0 +1,111 @@
+import os
+import metricbeat
+
+
+class TestCommands(metricbeat.BaseTest):
+    """
+    Test metricbeat subcommands
+    """
+
+    def setUp(self):
+        super(TestCommands, self).setUp()
+
+        # Enable modules reload with default paths
+        self.render_config_template(reload=True)
+        os.mkdir(self.working_dir + "/modules.d")
+
+    def test_modules_list(self):
+        """
+        Test modules list command
+        """
+        self.touch(self.working_dir + "/modules.d/enabled.yml")
+        self.touch(self.working_dir + "/modules.d/disabled.yml.disabled")
+
+        exit_code = self.run_beat(extra_args=["modules", "list"])
+
+        assert exit_code == 0
+        assert "Enabled:\nenabled" in self.get_log()
+        assert "Disabled:\ndisabled" in self.get_log()
+
+        # Add one more disabled module
+        self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled")
+        exit_code = self.run_beat(extra_args=["modules", "list"])
+
+        assert exit_code == 0
+        assert "Enabled:\nenabled" in self.get_log()
+        assert "Disabled:\ndisabled\ndisabled2" in self.get_log()
+
+    def test_modules_enable(self):
+        """
+        Test modules enable command
+        """
+        self.touch(self.working_dir + "/modules.d/enabled.yml")
+        self.touch(self.working_dir + "/modules.d/disabled1.yml.disabled")
+        self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled")
+        self.touch(self.working_dir + "/modules.d/disabled3.yml.disabled")
+
+        # Enable one module
+        exit_code = self.run_beat(
+            extra_args=["modules", "enable", "disabled1"])
+        assert exit_code == 0
+
+        assert self.log_contains("Enabled disabled1")
+        assert os.path.exists(self.working_dir + "/modules.d/disabled1.yml")
+        assert not os.path.exists(
+            self.working_dir + "/modules.d/disabled1.yml.disabled")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/disabled2.yml.disabled")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/disabled3.yml.disabled")
+
+        # Enable several modules at once:
+        exit_code = self.run_beat(
+            extra_args=["modules", "enable", "disabled2", "disabled3"])
+        assert exit_code == 0
+
+        assert self.log_contains("Enabled disabled2")
+        assert self.log_contains("Enabled disabled3")
+        assert os.path.exists(self.working_dir + "/modules.d/disabled2.yml")
+        assert os.path.exists(self.working_dir + "/modules.d/disabled3.yml")
+        assert not os.path.exists(
+            self.working_dir + "/modules.d/disabled2.yml.disabled")
+        assert not os.path.exists(
+            self.working_dir + "/modules.d/disabled3.yml.disabled")
+
+    def test_modules_disable(self):
+        """
+        Test modules disable command
+        """
+        self.touch(self.working_dir + "/modules.d/enabled1.yml")
+        self.touch(self.working_dir + "/modules.d/enabled2.yml")
+        self.touch(self.working_dir + "/modules.d/enabled3.yml")
+        self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled")
+
+        # Disable one module
+        exit_code = self.run_beat(
+            extra_args=["modules", "disable", "enabled1"])
+        assert exit_code == 0
+
+        assert self.log_contains("Disabled enabled1")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/enabled1.yml.disabled")
+        assert not os.path.exists(self.working_dir + "/modules.d/enabled1.yml")
+        assert os.path.exists(self.working_dir + "/modules.d/enabled2.yml")
+        assert os.path.exists(self.working_dir + "/modules.d/enabled3.yml")
+
+        # Disable several modules at once:
+        exit_code = self.run_beat(
+            extra_args=["modules", "disable", "enabled2", "enabled3"])
+        assert exit_code == 0
+
+        assert self.log_contains("Disabled enabled2")
+        assert self.log_contains("Disabled enabled3")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/enabled2.yml.disabled")
+        assert os.path.exists(
+            self.working_dir + "/modules.d/enabled3.yml.disabled")
+        assert not os.path.exists(self.working_dir + "/modules.d/enabled2.yml")
+        assert not os.path.exists(self.working_dir + "/modules.d/enabled3.yml")
+
+    def touch(self, path):
+        open(path, 'a').close()

From 071f106693a0437b3809426f11507b8629986351 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Tue, 4 Jul 2017 18:52:38 +0200
Subject: [PATCH 10/14] Update packaging to include `modules.d` folders

---
 dev-tools/packer/platforms/binary/run.sh.j2  |  1 +
 dev-tools/packer/platforms/centos/run.sh.j2  | 42 +++++++++++---------
 dev-tools/packer/platforms/darwin/run.sh.j2  |  1 +
 dev-tools/packer/platforms/debian/run.sh.j2  | 42 +++++++++++---------
 dev-tools/packer/platforms/windows/run.sh.j2 |  1 +
 dev-tools/packer/xgo-scripts/before_build.sh |  7 +++-
 metricbeat/Makefile                          |  4 +-
 7 files changed, 58 insertions(+), 40 deletions(-)

diff --git a/dev-tools/packer/platforms/binary/run.sh.j2 b/dev-tools/packer/platforms/binary/run.sh.j2
index f1caab8e90d..c52db2c5f46 100644
--- a/dev-tools/packer/platforms/binary/run.sh.j2
+++ b/dev-tools/packer/platforms/binary/run.sh.j2
@@ -16,6 +16,7 @@ cp {{.beat_name}}-linux-{{.arch}} /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}
 cp {{.beat_name}}-linux.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.yml
 cp {{.beat_name}}-linux.reference.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.reference.yml
 cp fields.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
+cp -a modules.d-linux/ /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/modules.d || true
 
 mkdir -p upload
 tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}
diff --git a/dev-tools/packer/platforms/centos/run.sh.j2 b/dev-tools/packer/platforms/centos/run.sh.j2
index c83313dee2b..5a5ca7f95af 100644
--- a/dev-tools/packer/platforms/centos/run.sh.j2
+++ b/dev-tools/packer/platforms/centos/run.sh.j2
@@ -21,26 +21,30 @@ fi
 RPM_VERSION=`echo ${VERSION} | sed 's/-/_/g'`
 
 # create rpm
-fpm --force -s dir -t rpm \
-        -n {{.beat_pkg_name}} -v ${RPM_VERSION} \
-        --architecture {{.rpm_arch}} \
-        --vendor "{{.beat_vendor}}"  \
-        --license "{{.beat_license}}"  \
-        --description "{{.beat_description}}" \
-        --url {{.beat_url}} \
-        --rpm-init /tmp/{{.beat_pkg_name}}.init \
-        --after-install /tmp/systemd-daemon-reload.sh \
-        --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml \
-        homedir/=/usr/share/{{.beat_name}} \
-        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh \
-        {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
-        {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
-        {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml \
-        fields.yml=/etc/{{.beat_name}}/fields.yml \
-        ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service \
-        god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
-
+COMMAND='fpm --force -s dir -t rpm
+        -n {{.beat_pkg_name}} -v ${RPM_VERSION}
+        --architecture {{.rpm_arch}}
+        --vendor "{{.beat_vendor}}"
+        --license "{{.beat_license}}"
+        --description "{{.beat_description}}"
+        --url {{.beat_url}}
+        --rpm-init /tmp/{{.beat_pkg_name}}.init
+        --after-install /tmp/systemd-daemon-reload.sh
+        --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
+        homedir/=/usr/share/{{.beat_name}}
+        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
+        {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
+        {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
+        {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml
+        fields.yml=/etc/{{.beat_name}}/fields.yml
+        ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service
+        god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god'
+
+if [ -d modules.d-linux ]; then
+  COMMAND="$COMMAND modules.d-linux/=/etc/{{.beat_name}}/modules.d/"
+fi
 
+eval $COMMAND
 
 # rename so that the filename respects semver rules
 mkdir -p upload
diff --git a/dev-tools/packer/platforms/darwin/run.sh.j2 b/dev-tools/packer/platforms/darwin/run.sh.j2
index a51bbd6a88d..c0bedf7a2d3 100644
--- a/dev-tools/packer/platforms/darwin/run.sh.j2
+++ b/dev-tools/packer/platforms/darwin/run.sh.j2
@@ -16,6 +16,7 @@ cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_
 cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.yml
 cp {{.beat_name}}-darwin.reference.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.reference.yml
 cp fields.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/
+cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-darwin-x86_64/modules.d || true
 
 mkdir -p upload
 tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64
diff --git a/dev-tools/packer/platforms/debian/run.sh.j2 b/dev-tools/packer/platforms/debian/run.sh.j2
index b512fdd592e..634317a6d70 100644
--- a/dev-tools/packer/platforms/debian/run.sh.j2
+++ b/dev-tools/packer/platforms/debian/run.sh.j2
@@ -18,24 +18,30 @@ if [ "$SNAPSHOT" = "yes" ]; then
 fi
 
 # create deb
-fpm --force -s dir -t deb \
-        -n {{.beat_pkg_name}} -v ${VERSION} \
-        --vendor "{{.beat_vendor}}"  \
-        --license "{{.beat_license}}"  \
-        --architecture {{.deb_arch}} \
-        --description "{{.beat_description}}" \
-        --url {{.beat_url}} \
-        --deb-init /tmp/{{.beat_pkg_name}}.init \
-        --after-install /tmp/systemd-daemon-reload.sh \
-        --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml \
-        homedir/=/usr/share/{{.beat_name}} \
-        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh \
-        {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
-        {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
-        {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml \
-        fields.yml=/etc/{{.beat_name}}/fields.yml \
-        ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service \
-        god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
+COMMAND='fpm --force -s dir -t deb
+        -n {{.beat_pkg_name}} -v ${VERSION}
+        --vendor "{{.beat_vendor}}"
+        --license "{{.beat_license}}"
+        --architecture {{.deb_arch}}
+        --description "{{.beat_description}}"
+        --url {{.beat_url}}
+        --deb-init /tmp/{{.beat_pkg_name}}.init
+        --after-install /tmp/systemd-daemon-reload.sh
+        --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
+        homedir/=/usr/share/{{.beat_name}}
+        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
+        {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
+        {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
+        {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml
+        fields.yml=/etc/{{.beat_name}}/fields.yml
+        ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service
+        god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god'
+
+if [ -d modules.d-linux ]; then
+  COMMAND="$COMMAND modules.d-linux/=/etc/{{.beat_name}}/modules.d/"
+fi
+
+eval $COMMAND
 
 # move and rename to use the elastic conventions
 mkdir -p upload
diff --git a/dev-tools/packer/platforms/windows/run.sh.j2 b/dev-tools/packer/platforms/windows/run.sh.j2
index 4f19a6aebc3..dee7a536ae7 100644
--- a/dev-tools/packer/platforms/windows/run.sh.j2
+++ b/dev-tools/packer/platforms/windows/run.sh.j2
@@ -17,6 +17,7 @@ unix2dos {{.beat_name}}-win.yml
 cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.yml
 cp {{.beat_name}}-win.reference.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.reference.yml
 cp fields.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
+cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/modules.d || true
 cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 
diff --git a/dev-tools/packer/xgo-scripts/before_build.sh b/dev-tools/packer/xgo-scripts/before_build.sh
index aa2fdd87288..6b4c94f02f5 100755
--- a/dev-tools/packer/xgo-scripts/before_build.sh
+++ b/dev-tools/packer/xgo-scripts/before_build.sh
@@ -19,16 +19,22 @@ cp fields.yml $PREFIX/fields.yml
 cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml
 chmod 0600 $PREFIX/$BEAT_NAME-linux.yml
 cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-linux.reference.yml
+rm -rf $PREFIX/modules.d-linux
+cp -a modules.d/ $PREFIX/modules.d-linux || true
 
 # darwin
 cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-darwin.yml
 chmod 0600 $PREFIX/$BEAT_NAME-darwin.yml
 cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-darwin.reference.yml
+rm -rf $PREFIX/modules.d-darwin
+cp -a modules.d/ $PREFIX/modules.d-darwin || true
 
 # win
 cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-win.yml
 chmod 0600 $PREFIX/$BEAT_NAME-win.yml
 cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-win.reference.yml
+rm -rf $PREFIX/modules.d-win
+cp -a modules.d/ $PREFIX/modules.d-win || true
 
 # Runs beat specific tasks which should be done before building
 PREFIX=$PREFIX make before-build
@@ -49,4 +55,3 @@ sed -i -e 's/:doc-branch/doc_branch/g' ${PREFIX}/package.yml
 
 # Create README file
 /go/bin/gotpl /templates/readme.md.j2 < ${PREFIX}/package.yml > ${PREFIX}/homedir/README.md
-
diff --git a/metricbeat/Makefile b/metricbeat/Makefile
index 9b2eabe5967..f89bc07ac27 100644
--- a/metricbeat/Makefile
+++ b/metricbeat/Makefile
@@ -67,8 +67,8 @@ imports: python-env
 before-build:
 ifeq ($(BEAT_NAME), metricbeat)
 	# disable the system/load metricset on windows
-	sed -i.bk 's/- load/#- load/' $(PREFIX)/metricbeat-win.yml
-	rm $(PREFIX)/metricbeat-win.yml.bk
+	sed -i.bk 's/- load/#- load/' $(PREFIX)/modules.d-win/system.yml
+	rm $(PREFIX)/modules.d-win/system.yml.bk
 	sed -i.bk 's/- load/#- load/' $(PREFIX)/metricbeat-win.reference.yml
 	rm $(PREFIX)/metricbeat-win.reference.yml.bk
 endif

From cb2eef4a671d5b01cd6ce95c289352bf50f2a6c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Wed, 5 Jul 2017 11:59:05 +0200
Subject: [PATCH 11/14] Use `$FPM_ARGS` instead of `$COMMAND`

---
 dev-tools/packer/platforms/centos/run.sh.j2 | 10 ++++++----
 dev-tools/packer/platforms/debian/run.sh.j2 | 10 ++++++----
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/dev-tools/packer/platforms/centos/run.sh.j2 b/dev-tools/packer/platforms/centos/run.sh.j2
index 5a5ca7f95af..3ac232ce569 100644
--- a/dev-tools/packer/platforms/centos/run.sh.j2
+++ b/dev-tools/packer/platforms/centos/run.sh.j2
@@ -21,7 +21,8 @@ fi
 RPM_VERSION=`echo ${VERSION} | sed 's/-/_/g'`
 
 # create rpm
-COMMAND='fpm --force -s dir -t rpm
+FPM_ARGS=(
+        --force -s dir -t rpm
         -n {{.beat_pkg_name}} -v ${RPM_VERSION}
         --architecture {{.rpm_arch}}
         --vendor "{{.beat_vendor}}"
@@ -38,13 +39,14 @@ COMMAND='fpm --force -s dir -t rpm
         {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml
         fields.yml=/etc/{{.beat_name}}/fields.yml
         ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service
-        god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god'
+        god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
+  )
 
 if [ -d modules.d-linux ]; then
-  COMMAND="$COMMAND modules.d-linux/=/etc/{{.beat_name}}/modules.d/"
+  FPM_ARGS+=(modules.d-linux/=/etc/{{.beat_name}}/modules.d/)
 fi
 
-eval $COMMAND
+fpm "${FPM_ARGS[@]}"
 
 # rename so that the filename respects semver rules
 mkdir -p upload
diff --git a/dev-tools/packer/platforms/debian/run.sh.j2 b/dev-tools/packer/platforms/debian/run.sh.j2
index 634317a6d70..11f88d0cc7b 100644
--- a/dev-tools/packer/platforms/debian/run.sh.j2
+++ b/dev-tools/packer/platforms/debian/run.sh.j2
@@ -18,7 +18,8 @@ if [ "$SNAPSHOT" = "yes" ]; then
 fi
 
 # create deb
-COMMAND='fpm --force -s dir -t deb
+FPM_ARGS=(
+        --force -s dir -t deb
         -n {{.beat_pkg_name}} -v ${VERSION}
         --vendor "{{.beat_vendor}}"
         --license "{{.beat_license}}"
@@ -35,13 +36,14 @@ COMMAND='fpm --force -s dir -t deb
         {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml
         fields.yml=/etc/{{.beat_name}}/fields.yml
         ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service
-        god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god'
+        god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
+  )
 
 if [ -d modules.d-linux ]; then
-  COMMAND="$COMMAND modules.d-linux/=/etc/{{.beat_name}}/modules.d/"
+  FPM_ARGS+=(modules.d-linux/=/etc/{{.beat_name}}/modules.d/)
 fi
 
-eval $COMMAND
+fpm "${FPM_ARGS[@]}"
 
 # move and rename to use the elastic conventions
 mkdir -p upload

From d9e9a203346addf045ac5cef929c971c60c1c565 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Wed, 5 Jul 2017 13:20:41 +0200
Subject: [PATCH 12/14] Uncomment filebeat modules

As we moved them into modules.d, they won't be enabled until the file is
renamed
---
 filebeat/module/apache2/_meta/config.yml | 10 +++++-----
 filebeat/module/auditd/_meta/config.yml  |  7 +++----
 filebeat/module/icinga/_meta/config.yml  | 14 +++++++-------
 filebeat/module/mysql/_meta/config.yml   | 10 +++++-----
 filebeat/module/nginx/_meta/config.yml   | 10 +++++-----
 filebeat/module/system/_meta/config.yml  | 10 +++++-----
 filebeat/modules.d/apache2.yml.disabled  | 10 +++++-----
 filebeat/modules.d/auditd.yml.disabled   |  7 +++----
 filebeat/modules.d/icinga.yml.disabled   | 14 +++++++-------
 filebeat/modules.d/mysql.yml.disabled    | 10 +++++-----
 filebeat/modules.d/nginx.yml.disabled    | 10 +++++-----
 filebeat/modules.d/system.yml.disabled   | 10 +++++-----
 12 files changed, 60 insertions(+), 62 deletions(-)

diff --git a/filebeat/module/apache2/_meta/config.yml b/filebeat/module/apache2/_meta/config.yml
index 6ccb548ad4f..6a545ad3645 100644
--- a/filebeat/module/apache2/_meta/config.yml
+++ b/filebeat/module/apache2/_meta/config.yml
@@ -1,15 +1,15 @@
-#- module: apache2
+- module: apache2
   # Access logs
-  #access:
-    #enabled: true
+  access:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Error logs
-  #error:
-    #enabled: true
+  error:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/module/auditd/_meta/config.yml b/filebeat/module/auditd/_meta/config.yml
index 5669924ba0b..bd952f49cc9 100644
--- a/filebeat/module/auditd/_meta/config.yml
+++ b/filebeat/module/auditd/_meta/config.yml
@@ -1,8 +1,7 @@
-#- module: auditd
-  #log:
-    #enabled: true
+- module: auditd
+  log:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
-
diff --git a/filebeat/module/icinga/_meta/config.yml b/filebeat/module/icinga/_meta/config.yml
index 7657f4467ee..afcd57986a2 100644
--- a/filebeat/module/icinga/_meta/config.yml
+++ b/filebeat/module/icinga/_meta/config.yml
@@ -1,23 +1,23 @@
-#- module: icinga
+- module: icinga
   # Main logs
-  #main:
-    #enabled: true
+  main:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Debug logs
-  #debug:
-    #enabled: true
+  debug:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Startup logs
-  #startup:
-    #enabled: true
+  startup:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/module/mysql/_meta/config.yml b/filebeat/module/mysql/_meta/config.yml
index aadb5631092..10afcb9e0ab 100644
--- a/filebeat/module/mysql/_meta/config.yml
+++ b/filebeat/module/mysql/_meta/config.yml
@@ -1,15 +1,15 @@
-#- module: mysql
+- module: mysql
   # Error logs
-  #error:
-    #enabled: true
+  error:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Slow logs
-  #slowlog:
-    #enabled: true
+  slowlog:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/module/nginx/_meta/config.yml b/filebeat/module/nginx/_meta/config.yml
index e1b45939726..22fe90e486e 100644
--- a/filebeat/module/nginx/_meta/config.yml
+++ b/filebeat/module/nginx/_meta/config.yml
@@ -1,15 +1,15 @@
-#- module: nginx
+- module: nginx
   # Access logs
-  #access:
-    #enabled: true
+  access:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Error logs
-  #error:
-    #enabled: true
+  error:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml
index d31ca8bc6ea..f76dd905b4d 100644
--- a/filebeat/module/system/_meta/config.yml
+++ b/filebeat/module/system/_meta/config.yml
@@ -1,15 +1,15 @@
-#- module: system
+- module: system
   # Syslog
-  #syslog:
-    #enabled: true
+  syslog:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Authorization logs
-  #auth:
-    #enabled: true
+  auth:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/modules.d/apache2.yml.disabled b/filebeat/modules.d/apache2.yml.disabled
index 6ccb548ad4f..6a545ad3645 100644
--- a/filebeat/modules.d/apache2.yml.disabled
+++ b/filebeat/modules.d/apache2.yml.disabled
@@ -1,15 +1,15 @@
-#- module: apache2
+- module: apache2
   # Access logs
-  #access:
-    #enabled: true
+  access:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Error logs
-  #error:
-    #enabled: true
+  error:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/modules.d/auditd.yml.disabled b/filebeat/modules.d/auditd.yml.disabled
index 5669924ba0b..bd952f49cc9 100644
--- a/filebeat/modules.d/auditd.yml.disabled
+++ b/filebeat/modules.d/auditd.yml.disabled
@@ -1,8 +1,7 @@
-#- module: auditd
-  #log:
-    #enabled: true
+- module: auditd
+  log:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
-
diff --git a/filebeat/modules.d/icinga.yml.disabled b/filebeat/modules.d/icinga.yml.disabled
index 7657f4467ee..afcd57986a2 100644
--- a/filebeat/modules.d/icinga.yml.disabled
+++ b/filebeat/modules.d/icinga.yml.disabled
@@ -1,23 +1,23 @@
-#- module: icinga
+- module: icinga
   # Main logs
-  #main:
-    #enabled: true
+  main:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Debug logs
-  #debug:
-    #enabled: true
+  debug:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Startup logs
-  #startup:
-    #enabled: true
+  startup:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/modules.d/mysql.yml.disabled b/filebeat/modules.d/mysql.yml.disabled
index aadb5631092..10afcb9e0ab 100644
--- a/filebeat/modules.d/mysql.yml.disabled
+++ b/filebeat/modules.d/mysql.yml.disabled
@@ -1,15 +1,15 @@
-#- module: mysql
+- module: mysql
   # Error logs
-  #error:
-    #enabled: true
+  error:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Slow logs
-  #slowlog:
-    #enabled: true
+  slowlog:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/modules.d/nginx.yml.disabled b/filebeat/modules.d/nginx.yml.disabled
index e1b45939726..22fe90e486e 100644
--- a/filebeat/modules.d/nginx.yml.disabled
+++ b/filebeat/modules.d/nginx.yml.disabled
@@ -1,15 +1,15 @@
-#- module: nginx
+- module: nginx
   # Access logs
-  #access:
-    #enabled: true
+  access:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Error logs
-  #error:
-    #enabled: true
+  error:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled
index d31ca8bc6ea..f76dd905b4d 100644
--- a/filebeat/modules.d/system.yml.disabled
+++ b/filebeat/modules.d/system.yml.disabled
@@ -1,15 +1,15 @@
-#- module: system
+- module: system
   # Syslog
-  #syslog:
-    #enabled: true
+  syslog:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Authorization logs
-  #auth:
-    #enabled: true
+  auth:
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.

From 44aab7428d2b65b6238745ab7108fa8e82c1b7d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Wed, 5 Jul 2017 15:32:47 +0200
Subject: [PATCH 13/14] Do not fail if ES connection fails on pipeline loading

---
 filebeat/beater/filebeat.go                  | 23 ++++---
 filebeat/crawler/crawler.go                  |  5 +-
 filebeat/fileset/factory.go                  | 63 +++++++++++---------
 filebeat/fileset/modules.go                  |  3 +
 filebeat/tests/system/test_reload_modules.py | 27 ++++++++-
 5 files changed, 83 insertions(+), 38 deletions(-)

diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go
index a5927c1a4ef..c8921680bc2 100644
--- a/filebeat/beater/filebeat.go
+++ b/filebeat/beater/filebeat.go
@@ -221,18 +221,15 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
 		spooler.Stop()
 	}()
 
-	var esClient fileset.PipelineLoader
+	// Create a ES connection factory for dynamic modules pipeline loading
+	var pipelineLoaderFactory fileset.PipelineLoaderFactory
 	if b.Config.Output.Name() == "elasticsearch" {
-		esConfig := b.Config.Output.Config()
-		esClient, err = elasticsearch.NewConnectedClient(esConfig)
-		if err != nil {
-			return errors.Wrap(err, "Error creating Elasticsearch client")
-		}
+		pipelineLoaderFactory = newPipelineLoaderFactory(b.Config.Output.Config())
 	} else {
 		logp.Warn(pipelinesWarning)
 	}
 
-	err = crawler.Start(registrar, config.ConfigProspector, config.ConfigModules, esClient)
+	err = crawler.Start(registrar, config.ConfigProspector, config.ConfigModules, pipelineLoaderFactory)
 	if err != nil {
 		crawler.Stop()
 		return err
@@ -285,3 +282,15 @@ func (fb *Filebeat) Stop() {
 	// Stop Filebeat
 	close(fb.done)
 }
+
+// Create a new pipeline loader (es client) factory
+func newPipelineLoaderFactory(esConfig *common.Config) fileset.PipelineLoaderFactory {
+	pipelineLoaderFactory := func() (fileset.PipelineLoader, error) {
+		esClient, err := elasticsearch.NewConnectedClient(esConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error creating Elasticsearch client")
+		}
+		return esClient, nil
+	}
+	return pipelineLoaderFactory
+}
diff --git a/filebeat/crawler/crawler.go b/filebeat/crawler/crawler.go
index 7bcecbd6f65..882cd22ee78 100644
--- a/filebeat/crawler/crawler.go
+++ b/filebeat/crawler/crawler.go
@@ -38,7 +38,8 @@ func New(out channel.Outleter, prospectorConfigs []*common.Config, beatVersion s
 }
 
 // Start starts the crawler with all prospectors
-func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config, configModules *common.Config, pipelineLoader fileset.PipelineLoader) error {
+func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config,
+	configModules *common.Config, pipelineLoaderFactory fileset.PipelineLoaderFactory) error {
 
 	logp.Info("Loading Prospectors: %v", len(c.prospectorConfigs))
 
@@ -64,7 +65,7 @@ func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config
 		logp.Beta("Loading separate modules is enabled.")
 
 		c.reloader = cfgfile.NewReloader(configModules)
-		factory := fileset.NewFactory(c.out, r, c.beatVersion, pipelineLoader, c.beatDone)
+		factory := fileset.NewFactory(c.out, r, c.beatVersion, pipelineLoaderFactory, c.beatDone)
 		go func() {
 			c.reloader.Run(factory)
 		}()
diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go
index 3a7791ff152..6e9b62b7dff 100644
--- a/filebeat/fileset/factory.go
+++ b/filebeat/fileset/factory.go
@@ -14,29 +14,30 @@ import (
 
 // Factory for modules
 type Factory struct {
-	outlet         channel.Outleter
-	registrar      *registrar.Registrar
-	beatVersion    string
-	pipelineLoader PipelineLoader
-	beatDone       chan struct{}
+	outlet                channel.Outleter
+	registrar             *registrar.Registrar
+	beatVersion           string
+	pipelineLoaderFactory PipelineLoaderFactory
+	beatDone              chan struct{}
 }
 
 // Wrap an array of prospectors and implements cfgfile.Runner interface
 type prospectorsRunner struct {
-	id             uint64
-	moduleRegistry *ModuleRegistry
-	prospectors    []*prospector.Prospector
-	pipelineLoader PipelineLoader
+	id                    uint64
+	moduleRegistry        *ModuleRegistry
+	prospectors           []*prospector.Prospector
+	pipelineLoaderFactory PipelineLoaderFactory
 }
 
 // NewFactory instantiates a new Factory
-func NewFactory(outlet channel.Outleter, registrar *registrar.Registrar, beatVersion string, pipelineLoader PipelineLoader, beatDone chan struct{}) *Factory {
+func NewFactory(outlet channel.Outleter, registrar *registrar.Registrar, beatVersion string,
+	pipelineLoaderFactory PipelineLoaderFactory, beatDone chan struct{}) *Factory {
 	return &Factory{
-		outlet:         outlet,
-		registrar:      registrar,
-		beatVersion:    beatVersion,
-		beatDone:       beatDone,
-		pipelineLoader: pipelineLoader,
+		outlet:                outlet,
+		registrar:             registrar,
+		beatVersion:           beatVersion,
+		beatDone:              beatDone,
+		pipelineLoaderFactory: pipelineLoaderFactory,
 	}
 }
 
@@ -71,27 +72,33 @@ func (f *Factory) Create(c *common.Config) (cfgfile.Runner, error) {
 	}
 
 	return &prospectorsRunner{
-		id:             id,
-		moduleRegistry: m,
-		prospectors:    prospectors,
-		pipelineLoader: f.pipelineLoader,
+		id:                    id,
+		moduleRegistry:        m,
+		prospectors:           prospectors,
+		pipelineLoaderFactory: f.pipelineLoaderFactory,
 	}, nil
 }
 
 func (p *prospectorsRunner) Start() {
 	// Load pipelines
-	if p.pipelineLoader != nil {
-		// Setup a callback & load now too, as we are already connected
-		callback := func(esClient *elasticsearch.Client) error {
-			return p.moduleRegistry.LoadPipelines(p.pipelineLoader)
-		}
-		elasticsearch.RegisterConnectCallback(callback)
-
-		err := p.moduleRegistry.LoadPipelines(p.pipelineLoader)
+	if p.pipelineLoaderFactory != nil {
+		// Load pipelines instantly and then setup a callback for reconnections:
+		pipelineLoader, err := p.pipelineLoaderFactory()
 		if err != nil {
-			// Log error and continue
 			logp.Err("Error loading pipeline: %s", err)
+		} else {
+			err := p.moduleRegistry.LoadPipelines(pipelineLoader)
+			if err != nil {
+				// Log error and continue
+				logp.Err("Error loading pipeline: %s", err)
+			}
 		}
+
+		// Callback:
+		callback := func(esClient *elasticsearch.Client) error {
+			return p.moduleRegistry.LoadPipelines(esClient)
+		}
+		elasticsearch.RegisterConnectCallback(callback)
 	}
 
 	for _, prospector := range p.prospectors {
diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go
index ec9aa93b12b..05a16214f10 100644
--- a/filebeat/fileset/modules.go
+++ b/filebeat/fileset/modules.go
@@ -247,6 +247,9 @@ func (reg *ModuleRegistry) GetProspectorConfigs() ([]*common.Config, error) {
 	return result, nil
 }
 
+// PipelineLoader factory builds and returns a PipelineLoader
+type PipelineLoaderFactory func() (PipelineLoader, error)
+
 // PipelineLoader is a subset of the Elasticsearch client API capable of loading
 // the pipelines.
 type PipelineLoader interface {
diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py
index dcba40c967e..6f2da885e20 100644
--- a/filebeat/tests/system/test_reload_modules.py
+++ b/filebeat/tests/system/test_reload_modules.py
@@ -27,7 +27,9 @@ class Test(BaseTest):
 
     def setUp(self):
         super(BaseTest, self).setUp()
-        self.es = Elasticsearch([self.get_elasticsearch_url()])
+        if INTEGRATION_TESTS:
+            self.es = Elasticsearch([self.get_elasticsearch_url()])
+
         # Copy system module
         shutil.copytree(os.path.join("module", "test"),
                         os.path.join(self.working_dir, "module", "test"))
@@ -93,6 +95,29 @@ def test_reload_writes_pipeline(self):
                                     for key in self.es.transport.perform_request("GET", "/_ingest/pipeline/").keys()))
         proc.check_kill_and_wait()
 
+    def test_no_es_connection(self):
+        """
+        Test pipeline loading failures don't crash filebeat
+        """
+        self.render_config_template(
+            reload=True,
+            reload_path=self.working_dir + "/configs/*.yml",
+            reload_type="modules",
+            prospectors=False,
+            elasticsearch={"host": 'errorhost:9201'}
+        )
+
+        proc = self.start_beat()
+
+        os.mkdir(self.working_dir + "/configs/")
+        with open(self.working_dir + "/configs/system.yml.test", 'w') as f:
+            f.write(moduleConfigTemplate.format(self.working_dir + "/logs/*"))
+        os.rename(self.working_dir + "/configs/system.yml.test",
+                  self.working_dir + "/configs/system.yml")
+
+        self.wait_until(lambda: self.log_contains("Error loading pipeline: Error creating Elasticsearch client"))
+        proc.check_kill_and_wait(0)
+
     def test_start_stop(self):
         """
         Test basic modules start and stop

From 1fb3b5b631b0f3eaf8a08c0345251e9215d1725f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= <exekias@gmail.com>
Date: Fri, 7 Jul 2017 17:45:25 +0200
Subject: [PATCH 14/14] Address review comments

---
 filebeat/Makefile               | 2 +-
 filebeat/cmd/modules.go         | 2 +-
 libbeat/cfgfile/glob_manager.go | 9 ++++-----
 metricbeat/Makefile             | 4 ++--
 metricbeat/cmd/modules.go       | 2 +-
 5 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/filebeat/Makefile b/filebeat/Makefile
index 7b14fa28200..7ca0c0fb018 100644
--- a/filebeat/Makefile
+++ b/filebeat/Makefile
@@ -38,7 +38,7 @@ configs: python-env
 	@cp ${ES_BEATS}/filebeat/_meta/common.p1.yml _meta/beat.yml
 	@cat ${ES_BEATS}/filebeat/_meta/common.p2.yml >> _meta/beat.yml
 	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml > _meta/beat.reference.yml
-	@. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
+	@${PYTHON_ENV}/bin/python  ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
 	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p2.yml >> _meta/beat.reference.yml
 	@rm -rf modules.d && mkdir -p modules.d
 	@for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
diff --git a/filebeat/cmd/modules.go b/filebeat/cmd/modules.go
index 4bf888266f4..a1e1969ae76 100644
--- a/filebeat/cmd/modules.go
+++ b/filebeat/cmd/modules.go
@@ -21,7 +21,7 @@ func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
 		return nil, errors.Errorf("modules management requires 'filebeat.config.modules.path' setting")
 	}
 
-	if !strings.HasSuffix(glob, ".yml") {
+	if !strings.HasSuffix(glob, "*.yml") {
 		return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob)
 	}
 
diff --git a/libbeat/cfgfile/glob_manager.go b/libbeat/cfgfile/glob_manager.go
index c8168241085..b3ef2e7554e 100644
--- a/libbeat/cfgfile/glob_manager.go
+++ b/libbeat/cfgfile/glob_manager.go
@@ -46,7 +46,8 @@ func NewGlobManager(glob, enabledExtension, disabledExtension string) (*GlobMana
 }
 
 func (g *GlobManager) load() error {
-	g.files = []*cfgfile{}
+	// empty previous data
+	g.files = nil
 
 	// Load enabled
 	watcher := NewGlobWatcher(g.glob)
@@ -85,8 +86,7 @@ func (g *GlobManager) load() error {
 
 // ListEnabled conf files
 func (g *GlobManager) ListEnabled() []string {
-	names := []string{}
-
+	var names []string
 	for _, file := range g.files {
 		if file.enabled {
 			names = append(names, file.name)
@@ -98,8 +98,7 @@ func (g *GlobManager) ListEnabled() []string {
 
 // ListDisabled conf files
 func (g *GlobManager) ListDisabled() []string {
-	names := []string{}
-
+	var names []string
 	for _, file := range g.files {
 		if !file.enabled {
 			names = append(names, file.name)
diff --git a/metricbeat/Makefile b/metricbeat/Makefile
index f89bc07ac27..cc828aca35b 100644
--- a/metricbeat/Makefile
+++ b/metricbeat/Makefile
@@ -49,11 +49,11 @@ configs: python-env
 	@cp ${ES_BEATS}/metricbeat/_meta/common.yml _meta/beat.yml
 	@cat ${ES_BEATS}/metricbeat/_meta/setup.yml >> _meta/beat.yml
 	@cat ${ES_BEATS}/metricbeat/_meta/common.reference.yml > _meta/beat.reference.yml
-	@. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
+	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
 	@rm -rf modules.d && mkdir -p modules.d
 	@for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
 	@chmod go-w modules.d/*
-	# Enable system by default:
+	@# Enable system by default:
 	@if [ -f modules.d/system.yml.disabled ]; then mv modules.d/system.yml.disabled modules.d/system.yml; fi
 
 # Generates imports for all modules and metricsets
diff --git a/metricbeat/cmd/modules.go b/metricbeat/cmd/modules.go
index 0df5a50ce6a..f7a467e5154 100644
--- a/metricbeat/cmd/modules.go
+++ b/metricbeat/cmd/modules.go
@@ -21,7 +21,7 @@ func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
 		return nil, errors.Errorf("modules management requires 'metricbeat.config.modules.path' setting")
 	}
 
-	if !strings.HasSuffix(glob, ".yml") {
+	if !strings.HasSuffix(glob, "*.yml") {
 		return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob)
 	}