Move to a modules.d layout (#4537)

* Move module settings to `modules.d` folder

We only want to use modules.d layout for `metricbeat.yml`, while keeping
`metricbeat.reference.yml` as a single file with all config options

* Unify config reloader usage

* Add modules management command

* Enable `modules` command for metricbeat

* Fix windows tests

* Add filebeat modules.d support

* Do not fail if system.yml is not present

* Comment out `reload.period` settings to make clear it's disabled

* Add system tests

* Update packaging to include `modules.d` folders

* Use `$FPM_ARGS` instead of `$COMMAND`

* Uncomment filebeat modules

As we moved them into modules.d, they won't be enabled until the file is
renamed

* Do not fail if ES connection fails on pipeline loading

* Address review comments

dev-tools/packer/platforms/binary/run.sh.j2

@@ -16,6 +16,7 @@ cp {{.beat_name}}-linux-{{.arch}} /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}
 cp {{.beat_name}}-linux.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.yml
 cp {{.beat_name}}-linux.reference.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.reference.yml
 cp fields.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
+cp -a modules.d-linux/ /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/modules.d || true
 
 mkdir -p upload
 tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}

dev-tools/packer/platforms/centos/run.sh.j2

@@ -21,26 +21,32 @@ fi
 RPM_VERSION=`echo ${VERSION} | sed 's/-/_/g'`
 
 # create rpm
-fpm --force -s dir -t rpm \
-        -n {{.beat_pkg_name}} -v ${RPM_VERSION} \
-        --architecture {{.rpm_arch}} \
-        --vendor "{{.beat_vendor}}"  \
-        --license "{{.beat_license}}"  \
-        --description "{{.beat_description}}" \
-        --url {{.beat_url}} \
-        --rpm-init /tmp/{{.beat_pkg_name}}.init \
-        --after-install /tmp/systemd-daemon-reload.sh \
-        --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml \
-        homedir/=/usr/share/{{.beat_name}} \
-        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh \
-        {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
-        {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
-        {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml \
-        fields.yml=/etc/{{.beat_name}}/fields.yml \
-        ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service \
+FPM_ARGS=(
+        --force -s dir -t rpm
+        -n {{.beat_pkg_name}} -v ${RPM_VERSION}
+        --architecture {{.rpm_arch}}
+        --vendor "{{.beat_vendor}}"
+        --license "{{.beat_license}}"
+        --description "{{.beat_description}}"
+        --url {{.beat_url}}
+        --rpm-init /tmp/{{.beat_pkg_name}}.init
+        --after-install /tmp/systemd-daemon-reload.sh
+        --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
+        homedir/=/usr/share/{{.beat_name}}
+        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
+        {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
+        {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
+        {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml
+        fields.yml=/etc/{{.beat_name}}/fields.yml
+        ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service
         god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
+  )
 
+if [ -d modules.d-linux ]; then
+  FPM_ARGS+=(modules.d-linux/=/etc/{{.beat_name}}/modules.d/)
+fi
 
+fpm "${FPM_ARGS[@]}"
 
 # rename so that the filename respects semver rules
 mkdir -p upload

dev-tools/packer/platforms/darwin/run.sh.j2

@@ -16,6 +16,7 @@ cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_
 cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.yml
 cp {{.beat_name}}-darwin.reference.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.reference.yml
 cp fields.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/
+cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-darwin-x86_64/modules.d || true
 
 mkdir -p upload
 tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64

dev-tools/packer/platforms/debian/run.sh.j2

@@ -18,24 +18,32 @@ if [ "$SNAPSHOT" = "yes" ]; then
 fi
 
 # create deb
-fpm --force -s dir -t deb \
-        -n {{.beat_pkg_name}} -v ${VERSION} \
-        --vendor "{{.beat_vendor}}"  \
-        --license "{{.beat_license}}"  \
-        --architecture {{.deb_arch}} \
-        --description "{{.beat_description}}" \
-        --url {{.beat_url}} \
-        --deb-init /tmp/{{.beat_pkg_name}}.init \
-        --after-install /tmp/systemd-daemon-reload.sh \
-        --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml \
-        homedir/=/usr/share/{{.beat_name}} \
-        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh \
-        {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
-        {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
-        {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml \
-        fields.yml=/etc/{{.beat_name}}/fields.yml \
-        ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service \
+FPM_ARGS=(
+        --force -s dir -t deb
+        -n {{.beat_pkg_name}} -v ${VERSION}
+        --vendor "{{.beat_vendor}}"
+        --license "{{.beat_license}}"
+        --architecture {{.deb_arch}}
+        --description "{{.beat_description}}"
+        --url {{.beat_url}}
+        --deb-init /tmp/{{.beat_pkg_name}}.init
+        --after-install /tmp/systemd-daemon-reload.sh
+        --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml
+        homedir/=/usr/share/{{.beat_name}}
+        beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh
+        {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}
+        {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml
+        {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml
+        fields.yml=/etc/{{.beat_name}}/fields.yml
+        ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service
         god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god
+  )
+
+if [ -d modules.d-linux ]; then
+  FPM_ARGS+=(modules.d-linux/=/etc/{{.beat_name}}/modules.d/)
+fi
+
+fpm "${FPM_ARGS[@]}"
 
 # move and rename to use the elastic conventions
 mkdir -p upload

dev-tools/packer/platforms/windows/run.sh.j2

@@ -17,6 +17,7 @@ unix2dos {{.beat_name}}-win.yml
 cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.yml
 cp {{.beat_name}}-win.reference.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.reference.yml
 cp fields.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
+cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/modules.d || true
 cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 

dev-tools/packer/xgo-scripts/before_build.sh

@@ -19,16 +19,22 @@ cp fields.yml $PREFIX/fields.yml
 cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml
 chmod 0600 $PREFIX/$BEAT_NAME-linux.yml
 cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-linux.reference.yml
+rm -rf $PREFIX/modules.d-linux
+cp -a modules.d/ $PREFIX/modules.d-linux || true
 
 # darwin
 cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-darwin.yml
 chmod 0600 $PREFIX/$BEAT_NAME-darwin.yml
 cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-darwin.reference.yml
+rm -rf $PREFIX/modules.d-darwin
+cp -a modules.d/ $PREFIX/modules.d-darwin || true
 
 # win
 cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-win.yml
 chmod 0600 $PREFIX/$BEAT_NAME-win.yml
 cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-win.reference.yml
+rm -rf $PREFIX/modules.d-win
+cp -a modules.d/ $PREFIX/modules.d-win || true
 
 # Runs beat specific tasks which should be done before building
 PREFIX=$PREFIX make before-build
@@ -49,4 +55,3 @@ sed -i -e 's/:doc-branch/doc_branch/g' ${PREFIX}/package.yml
 
 # Create README file
 /go/bin/gotpl /templates/readme.md.j2 < ${PREFIX}/package.yml > ${PREFIX}/homedir/README.md
-

filebeat/Makefile

@@ -36,11 +36,13 @@ modules:
 .PHONY: configs
 configs: python-env
 	@cp ${ES_BEATS}/filebeat/_meta/common.p1.yml _meta/beat.yml
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml
 	@cat ${ES_BEATS}/filebeat/_meta/common.p2.yml >> _meta/beat.yml
-	@cp ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml _meta/beat.reference.yml
-	@${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
+	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml > _meta/beat.reference.yml
+	@${PYTHON_ENV}/bin/python  ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml
 	@cat ${ES_BEATS}/filebeat/_meta/common.reference.p2.yml >> _meta/beat.reference.yml
+	@rm -rf modules.d && mkdir -p modules.d
+	@for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done
+	@chmod go-w modules.d/*
 
 # Collects all module docs
 .PHONY: collect-docs

filebeat/_meta/common.p2.yml

@@ -53,3 +53,15 @@ filebeat.prospectors:
   # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
   #multiline.match: after
 
+
+#============================= Filebeat modules ===============================
+
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s

filebeat/beater/filebeat.go

@@ -221,18 +221,15 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
 		spooler.Stop()
 	}()
 
-	var esClient fileset.PipelineLoader
+	// Create a ES connection factory for dynamic modules pipeline loading
+	var pipelineLoaderFactory fileset.PipelineLoaderFactory
 	if b.Config.Output.Name() == "elasticsearch" {
-		esConfig := b.Config.Output.Config()
-		esClient, err = elasticsearch.NewConnectedClient(esConfig)
-		if err != nil {
-			return errors.Wrap(err, "Error creating Elasticsearch client")
-		}
+		pipelineLoaderFactory = newPipelineLoaderFactory(b.Config.Output.Config())
 	} else {
 		logp.Warn(pipelinesWarning)
 	}
 
-	err = crawler.Start(registrar, config.ConfigProspector, config.ConfigModules, esClient)
+	err = crawler.Start(registrar, config.ConfigProspector, config.ConfigModules, pipelineLoaderFactory)
 	if err != nil {
 		crawler.Stop()
 		return err
@@ -285,3 +282,15 @@ func (fb *Filebeat) Stop() {
 	// Stop Filebeat
 	close(fb.done)
 }
+
+// Create a new pipeline loader (es client) factory
+func newPipelineLoaderFactory(esConfig *common.Config) fileset.PipelineLoaderFactory {
+	pipelineLoaderFactory := func() (fileset.PipelineLoader, error) {
+		esClient, err := elasticsearch.NewConnectedClient(esConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error creating Elasticsearch client")
+		}
+		return esClient, nil
+	}
+	return pipelineLoaderFactory
+}

filebeat/cmd/modules.go

@@ -0,0 +1,33 @@
+package cmd
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/cfgfile"
+	"github.com/elastic/beats/libbeat/cmd"
+)
+
+func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
+	config, err := beat.BeatConfig()
+	if err != nil {
+		return nil, errors.Wrap(err, "initialization error")
+	}
+
+	glob, err := config.String("config.modules.path", -1)
+	if err != nil {
+		return nil, errors.Errorf("modules management requires 'filebeat.config.modules.path' setting")
+	}
+
+	if !strings.HasSuffix(glob, "*.yml") {
+		return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob)
+	}
+
+	modulesManager, err := cfgfile.NewGlobManager(glob, ".yml", ".disabled")
+	if err != nil {
+		return nil, errors.Wrap(err, "initialization error")
+	}
+	return modulesManager, nil
+}

filebeat/cmd/root.go

@@ -25,4 +25,5 @@ func init() {
 	RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M"))
 	RootCmd.ConfigTestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
 	RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
+	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
 }

filebeat/crawler/crawler.go

@@ -38,7 +38,8 @@ func New(out channel.Outleter, prospectorConfigs []*common.Config, beatVersion s
 }
 
 // Start starts the crawler with all prospectors
-func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config, configModules *common.Config, pipelineLoader fileset.PipelineLoader) error {
+func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config,
+	configModules *common.Config, pipelineLoaderFactory fileset.PipelineLoaderFactory) error {
 
 	logp.Info("Loading Prospectors: %v", len(c.prospectorConfigs))
 
@@ -64,7 +65,7 @@ func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config
 		logp.Beta("Loading separate modules is enabled.")
 
 		c.reloader = cfgfile.NewReloader(configModules)
-		factory := fileset.NewFactory(c.out, r, c.beatVersion, pipelineLoader, c.beatDone)
+		factory := fileset.NewFactory(c.out, r, c.beatVersion, pipelineLoaderFactory, c.beatDone)
 		go func() {
 			c.reloader.Run(factory)
 		}()

filebeat/filebeat.yml

@@ -7,93 +7,6 @@
 # You can find the full configuration reference here:
 # https://www.elastic.co/guide/en/beats/filebeat/index.html
 
-
-#==========================  Modules configuration ============================
-filebeat.modules:
-
-#------------------------------- System Module -------------------------------
-#- module: system
-  # Syslog
-  #syslog:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-  # Authorization logs
-  #auth:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-#------------------------------- Apache2 Module ------------------------------
-#- module: apache2
-  # Access logs
-  #access:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-  # Error logs
-  #error:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-#------------------------------- Auditd Module -------------------------------
-#- module: auditd
-  #log:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-
-#-------------------------------- MySQL Module -------------------------------
-#- module: mysql
-  # Error logs
-  #error:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-  # Slow logs
-  #slowlog:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-#-------------------------------- Nginx Module -------------------------------
-#- module: nginx
-  # Access logs
-  #access:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-  # Error logs
-  #error:
-    #enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:
-
-
 # For more available modules and options, please see the filebeat.reference.yml sample
 # configuration file.
 
@@ -150,6 +63,18 @@ filebeat.prospectors:
   #multiline.match: after
 
 
+#============================= Filebeat modules ===============================
+
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
 #================================ General =====================================
 
 # The name of the shipper that publishes the network data. It can be used to group