diff --git a/dev-tools/packer/platforms/binary/run.sh.j2 b/dev-tools/packer/platforms/binary/run.sh.j2 index f1caab8e90d9..c52db2c5f463 100644 --- a/dev-tools/packer/platforms/binary/run.sh.j2 +++ b/dev-tools/packer/platforms/binary/run.sh.j2 @@ -16,6 +16,7 @@ cp {{.beat_name}}-linux-{{.arch}} /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}} cp {{.beat_name}}-linux.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.yml cp {{.beat_name}}-linux.reference.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.reference.yml cp fields.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/ +cp -a modules.d-linux/ /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/modules.d || true mkdir -p upload tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}} diff --git a/dev-tools/packer/platforms/centos/run.sh.j2 b/dev-tools/packer/platforms/centos/run.sh.j2 index c83313dee2b9..3ac232ce5699 100644 --- a/dev-tools/packer/platforms/centos/run.sh.j2 +++ b/dev-tools/packer/platforms/centos/run.sh.j2 @@ -21,26 +21,32 @@ fi RPM_VERSION=`echo ${VERSION} | sed 's/-/_/g'` # create rpm -fpm --force -s dir -t rpm \ - -n {{.beat_pkg_name}} -v ${RPM_VERSION} \ - --architecture {{.rpm_arch}} \ - --vendor "{{.beat_vendor}}" \ - --license "{{.beat_license}}" \ - --description "{{.beat_description}}" \ - --url {{.beat_url}} \ - --rpm-init /tmp/{{.beat_pkg_name}}.init \ - --after-install /tmp/systemd-daemon-reload.sh \ - --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml \ - homedir/=/usr/share/{{.beat_name}} \ - beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh \ - {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \ - {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \ - {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml \ - fields.yml=/etc/{{.beat_name}}/fields.yml \ - ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service \ +FPM_ARGS=( + --force -s dir -t rpm + -n {{.beat_pkg_name}} -v ${RPM_VERSION} + --architecture {{.rpm_arch}} + --vendor "{{.beat_vendor}}" + --license "{{.beat_license}}" + --description "{{.beat_description}}" + --url {{.beat_url}} + --rpm-init /tmp/{{.beat_pkg_name}}.init + --after-install /tmp/systemd-daemon-reload.sh + --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml + homedir/=/usr/share/{{.beat_name}} + beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh + {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} + {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml + {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml + fields.yml=/etc/{{.beat_name}}/fields.yml + ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god + ) +if [ -d modules.d-linux ]; then + FPM_ARGS+=(modules.d-linux/=/etc/{{.beat_name}}/modules.d/) +fi +fpm "${FPM_ARGS[@]}" # rename so that the filename respects semver rules mkdir -p upload diff --git a/dev-tools/packer/platforms/darwin/run.sh.j2 b/dev-tools/packer/platforms/darwin/run.sh.j2 index a51bbd6a88d9..c0bedf7a2d35 100644 --- a/dev-tools/packer/platforms/darwin/run.sh.j2 +++ b/dev-tools/packer/platforms/darwin/run.sh.j2 @@ -16,6 +16,7 @@ cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_ cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.yml cp {{.beat_name}}-darwin.reference.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.reference.yml cp fields.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/ +cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-darwin-x86_64/modules.d || true mkdir -p upload tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64 diff --git a/dev-tools/packer/platforms/debian/run.sh.j2 b/dev-tools/packer/platforms/debian/run.sh.j2 index b512fdd592eb..11f88d0cc7bc 100644 --- a/dev-tools/packer/platforms/debian/run.sh.j2 +++ b/dev-tools/packer/platforms/debian/run.sh.j2 @@ -18,24 +18,32 @@ if [ "$SNAPSHOT" = "yes" ]; then fi # create deb -fpm --force -s dir -t deb \ - -n {{.beat_pkg_name}} -v ${VERSION} \ - --vendor "{{.beat_vendor}}" \ - --license "{{.beat_license}}" \ - --architecture {{.deb_arch}} \ - --description "{{.beat_description}}" \ - --url {{.beat_url}} \ - --deb-init /tmp/{{.beat_pkg_name}}.init \ - --after-install /tmp/systemd-daemon-reload.sh \ - --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml \ - homedir/=/usr/share/{{.beat_name}} \ - beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh \ - {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \ - {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \ - {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml \ - fields.yml=/etc/{{.beat_name}}/fields.yml \ - ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service \ +FPM_ARGS=( + --force -s dir -t deb + -n {{.beat_pkg_name}} -v ${VERSION} + --vendor "{{.beat_vendor}}" + --license "{{.beat_license}}" + --architecture {{.deb_arch}} + --description "{{.beat_description}}" + --url {{.beat_url}} + --deb-init /tmp/{{.beat_pkg_name}}.init + --after-install /tmp/systemd-daemon-reload.sh + --config-files /etc/{{.beat_name}}/{{.beat_name}}.yml + homedir/=/usr/share/{{.beat_name}} + beatname-${RUNID}.sh=/usr/bin/{{.beat_name}}.sh + {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} + {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml + {{.beat_name}}-linux.reference.yml=/etc/{{.beat_name}}/{{.beat_name}}.reference.yml + fields.yml=/etc/{{.beat_name}}/fields.yml + ${RUNID}.service=/lib/systemd/system/{{.beat_pkg_name}}.service god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god + ) + +if [ -d modules.d-linux ]; then + FPM_ARGS+=(modules.d-linux/=/etc/{{.beat_name}}/modules.d/) +fi + +fpm "${FPM_ARGS[@]}" # move and rename to use the elastic conventions mkdir -p upload diff --git a/dev-tools/packer/platforms/windows/run.sh.j2 b/dev-tools/packer/platforms/windows/run.sh.j2 index 4f19a6aebc3b..dee7a536ae78 100644 --- a/dev-tools/packer/platforms/windows/run.sh.j2 +++ b/dev-tools/packer/platforms/windows/run.sh.j2 @@ -17,6 +17,7 @@ unix2dos {{.beat_name}}-win.yml cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.yml cp {{.beat_name}}-win.reference.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.reference.yml cp fields.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ +cp -a modules.d-darwin/ /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/modules.d || true cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ diff --git a/dev-tools/packer/xgo-scripts/before_build.sh b/dev-tools/packer/xgo-scripts/before_build.sh index aa2fdd87288e..6b4c94f02f57 100755 --- a/dev-tools/packer/xgo-scripts/before_build.sh +++ b/dev-tools/packer/xgo-scripts/before_build.sh @@ -19,16 +19,22 @@ cp fields.yml $PREFIX/fields.yml cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml chmod 0600 $PREFIX/$BEAT_NAME-linux.yml cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-linux.reference.yml +rm -rf $PREFIX/modules.d-linux +cp -a modules.d/ $PREFIX/modules.d-linux || true # darwin cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-darwin.yml chmod 0600 $PREFIX/$BEAT_NAME-darwin.yml cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-darwin.reference.yml +rm -rf $PREFIX/modules.d-darwin +cp -a modules.d/ $PREFIX/modules.d-darwin || true # win cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-win.yml chmod 0600 $PREFIX/$BEAT_NAME-win.yml cp $BEAT_NAME.reference.yml $PREFIX/$BEAT_NAME-win.reference.yml +rm -rf $PREFIX/modules.d-win +cp -a modules.d/ $PREFIX/modules.d-win || true # Runs beat specific tasks which should be done before building PREFIX=$PREFIX make before-build @@ -49,4 +55,3 @@ sed -i -e 's/:doc-branch/doc_branch/g' ${PREFIX}/package.yml # Create README file /go/bin/gotpl /templates/readme.md.j2 < ${PREFIX}/package.yml > ${PREFIX}/homedir/README.md - diff --git a/filebeat/Makefile b/filebeat/Makefile index 9a3bf4b44d63..7ca0c0fb018d 100644 --- a/filebeat/Makefile +++ b/filebeat/Makefile @@ -36,11 +36,13 @@ modules: .PHONY: configs configs: python-env @cp ${ES_BEATS}/filebeat/_meta/common.p1.yml _meta/beat.yml - @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml @cat ${ES_BEATS}/filebeat/_meta/common.p2.yml >> _meta/beat.yml - @cp ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml _meta/beat.reference.yml - @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml + @cat ${ES_BEATS}/filebeat/_meta/common.reference.p1.yml > _meta/beat.reference.yml + @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml @cat ${ES_BEATS}/filebeat/_meta/common.reference.p2.yml >> _meta/beat.reference.yml + @rm -rf modules.d && mkdir -p modules.d + @for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done + @chmod go-w modules.d/* # Collects all module docs .PHONY: collect-docs diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/common.p2.yml index ca2306d617b9..873910954eee 100644 --- a/filebeat/_meta/common.p2.yml +++ b/filebeat/_meta/common.p2.yml @@ -53,3 +53,15 @@ filebeat.prospectors: # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash #multiline.match: after + +#============================= Filebeat modules =============================== + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go index a5927c1a4efd..c8921680bc22 100644 --- a/filebeat/beater/filebeat.go +++ b/filebeat/beater/filebeat.go @@ -221,18 +221,15 @@ func (fb *Filebeat) Run(b *beat.Beat) error { spooler.Stop() }() - var esClient fileset.PipelineLoader + // Create a ES connection factory for dynamic modules pipeline loading + var pipelineLoaderFactory fileset.PipelineLoaderFactory if b.Config.Output.Name() == "elasticsearch" { - esConfig := b.Config.Output.Config() - esClient, err = elasticsearch.NewConnectedClient(esConfig) - if err != nil { - return errors.Wrap(err, "Error creating Elasticsearch client") - } + pipelineLoaderFactory = newPipelineLoaderFactory(b.Config.Output.Config()) } else { logp.Warn(pipelinesWarning) } - err = crawler.Start(registrar, config.ConfigProspector, config.ConfigModules, esClient) + err = crawler.Start(registrar, config.ConfigProspector, config.ConfigModules, pipelineLoaderFactory) if err != nil { crawler.Stop() return err @@ -285,3 +282,15 @@ func (fb *Filebeat) Stop() { // Stop Filebeat close(fb.done) } + +// Create a new pipeline loader (es client) factory +func newPipelineLoaderFactory(esConfig *common.Config) fileset.PipelineLoaderFactory { + pipelineLoaderFactory := func() (fileset.PipelineLoader, error) { + esClient, err := elasticsearch.NewConnectedClient(esConfig) + if err != nil { + return nil, errors.Wrap(err, "Error creating Elasticsearch client") + } + return esClient, nil + } + return pipelineLoaderFactory +} diff --git a/filebeat/cmd/modules.go b/filebeat/cmd/modules.go new file mode 100644 index 000000000000..a1e1969ae76b --- /dev/null +++ b/filebeat/cmd/modules.go @@ -0,0 +1,33 @@ +package cmd + +import ( + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/cfgfile" + "github.com/elastic/beats/libbeat/cmd" +) + +func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) { + config, err := beat.BeatConfig() + if err != nil { + return nil, errors.Wrap(err, "initialization error") + } + + glob, err := config.String("config.modules.path", -1) + if err != nil { + return nil, errors.Errorf("modules management requires 'filebeat.config.modules.path' setting") + } + + if !strings.HasSuffix(glob, "*.yml") { + return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob) + } + + modulesManager, err := cfgfile.NewGlobManager(glob, ".yml", ".disabled") + if err != nil { + return nil, errors.Wrap(err, "initialization error") + } + return modulesManager, nil +} diff --git a/filebeat/cmd/root.go b/filebeat/cmd/root.go index 4b685b657ef8..6d44116b25e8 100644 --- a/filebeat/cmd/root.go +++ b/filebeat/cmd/root.go @@ -25,4 +25,5 @@ func init() { RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M")) RootCmd.ConfigTestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules")) RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules")) + RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager)) } diff --git a/filebeat/crawler/crawler.go b/filebeat/crawler/crawler.go index 7bcecbd6f65d..882cd22ee78d 100644 --- a/filebeat/crawler/crawler.go +++ b/filebeat/crawler/crawler.go @@ -38,7 +38,8 @@ func New(out channel.Outleter, prospectorConfigs []*common.Config, beatVersion s } // Start starts the crawler with all prospectors -func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config, configModules *common.Config, pipelineLoader fileset.PipelineLoader) error { +func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config, + configModules *common.Config, pipelineLoaderFactory fileset.PipelineLoaderFactory) error { logp.Info("Loading Prospectors: %v", len(c.prospectorConfigs)) @@ -64,7 +65,7 @@ func (c *Crawler) Start(r *registrar.Registrar, configProspectors *common.Config logp.Beta("Loading separate modules is enabled.") c.reloader = cfgfile.NewReloader(configModules) - factory := fileset.NewFactory(c.out, r, c.beatVersion, pipelineLoader, c.beatDone) + factory := fileset.NewFactory(c.out, r, c.beatVersion, pipelineLoaderFactory, c.beatDone) go func() { c.reloader.Run(factory) }() diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index d941334e6477..a49d550f818d 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -7,93 +7,6 @@ # You can find the full configuration reference here: # https://www.elastic.co/guide/en/beats/filebeat/index.html - -#========================== Modules configuration ============================ -filebeat.modules: - -#------------------------------- System Module ------------------------------- -#- module: system - # Syslog - #syslog: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Authorization logs - #auth: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - -#------------------------------- Apache2 Module ------------------------------ -#- module: apache2 - # Access logs - #access: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Error logs - #error: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - -#------------------------------- Auditd Module ------------------------------- -#- module: auditd - #log: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - -#-------------------------------- MySQL Module ------------------------------- -#- module: mysql - # Error logs - #error: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Slow logs - #slowlog: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - -#-------------------------------- Nginx Module ------------------------------- -#- module: nginx - # Access logs - #access: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # Error logs - #error: - #enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: - - # For more available modules and options, please see the filebeat.reference.yml sample # configuration file. @@ -150,6 +63,18 @@ filebeat.prospectors: #multiline.match: after +#============================= Filebeat modules =============================== + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + #================================ General ===================================== # The name of the shipper that publishes the network data. It can be used to group diff --git a/filebeat/fileset/factory.go b/filebeat/fileset/factory.go index 3a7791ff1526..6e9b62b7dff7 100644 --- a/filebeat/fileset/factory.go +++ b/filebeat/fileset/factory.go @@ -14,29 +14,30 @@ import ( // Factory for modules type Factory struct { - outlet channel.Outleter - registrar *registrar.Registrar - beatVersion string - pipelineLoader PipelineLoader - beatDone chan struct{} + outlet channel.Outleter + registrar *registrar.Registrar + beatVersion string + pipelineLoaderFactory PipelineLoaderFactory + beatDone chan struct{} } // Wrap an array of prospectors and implements cfgfile.Runner interface type prospectorsRunner struct { - id uint64 - moduleRegistry *ModuleRegistry - prospectors []*prospector.Prospector - pipelineLoader PipelineLoader + id uint64 + moduleRegistry *ModuleRegistry + prospectors []*prospector.Prospector + pipelineLoaderFactory PipelineLoaderFactory } // NewFactory instantiates a new Factory -func NewFactory(outlet channel.Outleter, registrar *registrar.Registrar, beatVersion string, pipelineLoader PipelineLoader, beatDone chan struct{}) *Factory { +func NewFactory(outlet channel.Outleter, registrar *registrar.Registrar, beatVersion string, + pipelineLoaderFactory PipelineLoaderFactory, beatDone chan struct{}) *Factory { return &Factory{ - outlet: outlet, - registrar: registrar, - beatVersion: beatVersion, - beatDone: beatDone, - pipelineLoader: pipelineLoader, + outlet: outlet, + registrar: registrar, + beatVersion: beatVersion, + beatDone: beatDone, + pipelineLoaderFactory: pipelineLoaderFactory, } } @@ -71,27 +72,33 @@ func (f *Factory) Create(c *common.Config) (cfgfile.Runner, error) { } return &prospectorsRunner{ - id: id, - moduleRegistry: m, - prospectors: prospectors, - pipelineLoader: f.pipelineLoader, + id: id, + moduleRegistry: m, + prospectors: prospectors, + pipelineLoaderFactory: f.pipelineLoaderFactory, }, nil } func (p *prospectorsRunner) Start() { // Load pipelines - if p.pipelineLoader != nil { - // Setup a callback & load now too, as we are already connected - callback := func(esClient *elasticsearch.Client) error { - return p.moduleRegistry.LoadPipelines(p.pipelineLoader) - } - elasticsearch.RegisterConnectCallback(callback) - - err := p.moduleRegistry.LoadPipelines(p.pipelineLoader) + if p.pipelineLoaderFactory != nil { + // Load pipelines instantly and then setup a callback for reconnections: + pipelineLoader, err := p.pipelineLoaderFactory() if err != nil { - // Log error and continue logp.Err("Error loading pipeline: %s", err) + } else { + err := p.moduleRegistry.LoadPipelines(pipelineLoader) + if err != nil { + // Log error and continue + logp.Err("Error loading pipeline: %s", err) + } } + + // Callback: + callback := func(esClient *elasticsearch.Client) error { + return p.moduleRegistry.LoadPipelines(esClient) + } + elasticsearch.RegisterConnectCallback(callback) } for _, prospector := range p.prospectors { diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go index ec9aa93b12b6..05a16214f100 100644 --- a/filebeat/fileset/modules.go +++ b/filebeat/fileset/modules.go @@ -247,6 +247,9 @@ func (reg *ModuleRegistry) GetProspectorConfigs() ([]*common.Config, error) { return result, nil } +// PipelineLoader factory builds and returns a PipelineLoader +type PipelineLoaderFactory func() (PipelineLoader, error) + // PipelineLoader is a subset of the Elasticsearch client API capable of loading // the pipelines. type PipelineLoader interface { diff --git a/filebeat/module/apache2/_meta/config.yml b/filebeat/module/apache2/_meta/config.yml index 6ccb548ad4f4..6a545ad3645b 100644 --- a/filebeat/module/apache2/_meta/config.yml +++ b/filebeat/module/apache2/_meta/config.yml @@ -1,15 +1,15 @@ -#- module: apache2 +- module: apache2 # Access logs - #access: - #enabled: true + access: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: # Error logs - #error: - #enabled: true + error: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/filebeat/module/auditd/_meta/config.yml b/filebeat/module/auditd/_meta/config.yml index 5669924ba0b6..bd952f49cc9d 100644 --- a/filebeat/module/auditd/_meta/config.yml +++ b/filebeat/module/auditd/_meta/config.yml @@ -1,8 +1,7 @@ -#- module: auditd - #log: - #enabled: true +- module: auditd + log: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: - diff --git a/filebeat/module/icinga/_meta/config.yml b/filebeat/module/icinga/_meta/config.yml index 7657f4467eef..afcd57986a2d 100644 --- a/filebeat/module/icinga/_meta/config.yml +++ b/filebeat/module/icinga/_meta/config.yml @@ -1,23 +1,23 @@ -#- module: icinga +- module: icinga # Main logs - #main: - #enabled: true + main: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: # Debug logs - #debug: - #enabled: true + debug: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: # Startup logs - #startup: - #enabled: true + startup: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/filebeat/module/mysql/_meta/config.yml b/filebeat/module/mysql/_meta/config.yml index aadb56310927..10afcb9e0ab6 100644 --- a/filebeat/module/mysql/_meta/config.yml +++ b/filebeat/module/mysql/_meta/config.yml @@ -1,15 +1,15 @@ -#- module: mysql +- module: mysql # Error logs - #error: - #enabled: true + error: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: # Slow logs - #slowlog: - #enabled: true + slowlog: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/filebeat/module/nginx/_meta/config.yml b/filebeat/module/nginx/_meta/config.yml index e1b459397261..22fe90e486e4 100644 --- a/filebeat/module/nginx/_meta/config.yml +++ b/filebeat/module/nginx/_meta/config.yml @@ -1,15 +1,15 @@ -#- module: nginx +- module: nginx # Access logs - #access: - #enabled: true + access: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: # Error logs - #error: - #enabled: true + error: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml index d31ca8bc6eaf..f76dd905b4d7 100644 --- a/filebeat/module/system/_meta/config.yml +++ b/filebeat/module/system/_meta/config.yml @@ -1,15 +1,15 @@ -#- module: system +- module: system # Syslog - #syslog: - #enabled: true + syslog: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: # Authorization logs - #auth: - #enabled: true + auth: + enabled: true # Set custom paths for the log files. If left empty, # Filebeat will choose the paths depending on your OS. diff --git a/filebeat/modules.d/apache2.yml.disabled b/filebeat/modules.d/apache2.yml.disabled new file mode 100644 index 000000000000..6a545ad3645b --- /dev/null +++ b/filebeat/modules.d/apache2.yml.disabled @@ -0,0 +1,16 @@ +- module: apache2 + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/modules.d/auditd.yml.disabled b/filebeat/modules.d/auditd.yml.disabled new file mode 100644 index 000000000000..bd952f49cc9d --- /dev/null +++ b/filebeat/modules.d/auditd.yml.disabled @@ -0,0 +1,7 @@ +- module: auditd + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/modules.d/icinga.yml.disabled b/filebeat/modules.d/icinga.yml.disabled new file mode 100644 index 000000000000..afcd57986a2d --- /dev/null +++ b/filebeat/modules.d/icinga.yml.disabled @@ -0,0 +1,24 @@ +- module: icinga + # Main logs + main: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Debug logs + debug: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Startup logs + startup: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/modules.d/mysql.yml.disabled b/filebeat/modules.d/mysql.yml.disabled new file mode 100644 index 000000000000..10afcb9e0ab6 --- /dev/null +++ b/filebeat/modules.d/mysql.yml.disabled @@ -0,0 +1,16 @@ +- module: mysql + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Slow logs + slowlog: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/modules.d/nginx.yml.disabled b/filebeat/modules.d/nginx.yml.disabled new file mode 100644 index 000000000000..22fe90e486e4 --- /dev/null +++ b/filebeat/modules.d/nginx.yml.disabled @@ -0,0 +1,16 @@ +- module: nginx + # Access logs + access: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Error logs + error: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled new file mode 100644 index 000000000000..f76dd905b4d7 --- /dev/null +++ b/filebeat/modules.d/system.yml.disabled @@ -0,0 +1,16 @@ +- module: system + # Syslog + syslog: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + + # Authorization logs + auth: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/filebeat/tests/system/test_cmd.py b/filebeat/tests/system/test_cmd.py new file mode 100644 index 000000000000..94540b52d78f --- /dev/null +++ b/filebeat/tests/system/test_cmd.py @@ -0,0 +1,115 @@ +import os +import filebeat + + +class TestCommands(filebeat.BaseTest): + """ + Test filebeat subcommands + """ + + def setUp(self): + super(TestCommands, self).setUp() + + # Enable modules reload with default paths + self.render_config_template( + reload=True, + reload_type="modules", + reload_path="${path.config}/modules.d/*.yml", + ) + os.mkdir(self.working_dir + "/modules.d") + + def test_modules_list(self): + """ + Test modules list command + """ + self.touch(self.working_dir + "/modules.d/enabled.yml") + self.touch(self.working_dir + "/modules.d/disabled.yml.disabled") + + exit_code = self.run_beat(extra_args=["modules", "list"]) + + assert exit_code == 0 + assert "Enabled:\nenabled" in self.get_log() + assert "Disabled:\ndisabled" in self.get_log() + + # Add one more disabled module + self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled") + exit_code = self.run_beat(extra_args=["modules", "list"]) + + assert exit_code == 0 + assert "Enabled:\nenabled" in self.get_log() + assert "Disabled:\ndisabled\ndisabled2" in self.get_log() + + def test_modules_enable(self): + """ + Test modules enable command + """ + self.touch(self.working_dir + "/modules.d/enabled.yml") + self.touch(self.working_dir + "/modules.d/disabled1.yml.disabled") + self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled") + self.touch(self.working_dir + "/modules.d/disabled3.yml.disabled") + + # Enable one module + exit_code = self.run_beat( + extra_args=["modules", "enable", "disabled1"]) + assert exit_code == 0 + + assert self.log_contains("Enabled disabled1") + assert os.path.exists(self.working_dir + "/modules.d/disabled1.yml") + assert not os.path.exists( + self.working_dir + "/modules.d/disabled1.yml.disabled") + assert os.path.exists( + self.working_dir + "/modules.d/disabled2.yml.disabled") + assert os.path.exists( + self.working_dir + "/modules.d/disabled3.yml.disabled") + + # Enable several modules at once: + exit_code = self.run_beat( + extra_args=["modules", "enable", "disabled2", "disabled3"]) + assert exit_code == 0 + + assert self.log_contains("Enabled disabled2") + assert self.log_contains("Enabled disabled3") + assert os.path.exists(self.working_dir + "/modules.d/disabled2.yml") + assert os.path.exists(self.working_dir + "/modules.d/disabled3.yml") + assert not os.path.exists( + self.working_dir + "/modules.d/disabled2.yml.disabled") + assert not os.path.exists( + self.working_dir + "/modules.d/disabled3.yml.disabled") + + def test_modules_disable(self): + """ + Test modules disable command + """ + self.touch(self.working_dir + "/modules.d/enabled1.yml") + self.touch(self.working_dir + "/modules.d/enabled2.yml") + self.touch(self.working_dir + "/modules.d/enabled3.yml") + self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled") + + # Disable one module + exit_code = self.run_beat( + extra_args=["modules", "disable", "enabled1"]) + assert exit_code == 0 + + assert self.log_contains("Disabled enabled1") + assert os.path.exists( + self.working_dir + "/modules.d/enabled1.yml.disabled") + assert not os.path.exists(self.working_dir + "/modules.d/enabled1.yml") + assert os.path.exists(self.working_dir + "/modules.d/enabled2.yml") + assert os.path.exists(self.working_dir + "/modules.d/enabled3.yml") + + # Disable several modules at once: + exit_code = self.run_beat( + extra_args=["modules", "disable", "enabled2", "enabled3"]) + assert exit_code == 0 + + assert self.log_contains("Disabled enabled2") + assert self.log_contains("Disabled enabled3") + assert os.path.exists( + self.working_dir + "/modules.d/enabled2.yml.disabled") + assert os.path.exists( + self.working_dir + "/modules.d/enabled3.yml.disabled") + assert not os.path.exists(self.working_dir + "/modules.d/enabled2.yml") + assert not os.path.exists(self.working_dir + "/modules.d/enabled3.yml") + + def touch(self, path): + open(path, 'a').close() diff --git a/filebeat/tests/system/test_reload_modules.py b/filebeat/tests/system/test_reload_modules.py index dcba40c967e5..6f2da885e204 100644 --- a/filebeat/tests/system/test_reload_modules.py +++ b/filebeat/tests/system/test_reload_modules.py @@ -27,7 +27,9 @@ class Test(BaseTest): def setUp(self): super(BaseTest, self).setUp() - self.es = Elasticsearch([self.get_elasticsearch_url()]) + if INTEGRATION_TESTS: + self.es = Elasticsearch([self.get_elasticsearch_url()]) + # Copy system module shutil.copytree(os.path.join("module", "test"), os.path.join(self.working_dir, "module", "test")) @@ -93,6 +95,29 @@ def test_reload_writes_pipeline(self): for key in self.es.transport.perform_request("GET", "/_ingest/pipeline/").keys())) proc.check_kill_and_wait() + def test_no_es_connection(self): + """ + Test pipeline loading failures don't crash filebeat + """ + self.render_config_template( + reload=True, + reload_path=self.working_dir + "/configs/*.yml", + reload_type="modules", + prospectors=False, + elasticsearch={"host": 'errorhost:9201'} + ) + + proc = self.start_beat() + + os.mkdir(self.working_dir + "/configs/") + with open(self.working_dir + "/configs/system.yml.test", 'w') as f: + f.write(moduleConfigTemplate.format(self.working_dir + "/logs/*")) + os.rename(self.working_dir + "/configs/system.yml.test", + self.working_dir + "/configs/system.yml") + + self.wait_until(lambda: self.log_contains("Error loading pipeline: Error creating Elasticsearch client")) + proc.check_kill_and_wait(0) + def test_start_stop(self): """ Test basic modules start and stop diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 1e6387ba2e18..ec4e77fdf13c 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -213,8 +213,8 @@ func (b *Beat) Init() error { return nil } -// load the beats config section -func (b *Beat) config() (*common.Config, error) { +// BeatConfig returns config section for this beat +func (b *Beat) BeatConfig() (*common.Config, error) { configName := strings.ToLower(b.Info.Beat) if b.RawConfig.HasField(configName) { sub, err := b.RawConfig.Child(configName, -1) @@ -231,7 +231,7 @@ func (b *Beat) config() (*common.Config, error) { // create and return the beater, this method also initializes all needed items, // including template registering, publisher, xpack monitoring func (b *Beat) createBeater(bt Creator) (Beater, error) { - sub, err := b.config() + sub, err := b.BeatConfig() if err != nil { return nil, err } diff --git a/libbeat/cfgfile/glob_manager.go b/libbeat/cfgfile/glob_manager.go new file mode 100644 index 000000000000..b3ef2e7554e3 --- /dev/null +++ b/libbeat/cfgfile/glob_manager.go @@ -0,0 +1,167 @@ +package cfgfile + +import ( + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +// GlobManager allows to manage a directory of conf files. Using a glob pattern +// to match them, this object will allow to switch their state between enabled +// and disabled +type GlobManager struct { + glob string + enabledExtension string + disabledExtension string + files []*cfgfile +} + +type cfgfile struct { + name string + path string + enabled bool +} + +// NewGlobManager takes a glob and enabled/disabled extensions and returns a GlobManager object. +// Parameters: +// - glob - matching conf files (ie: modules.d/*.yml) +// - enabledExtension - extension for enabled confs, must match the glob (ie: .yml) +// - disabledExtension - extension to append for disabled confs (ie: .disabled) +func NewGlobManager(glob, enabledExtension, disabledExtension string) (*GlobManager, error) { + if !strings.HasSuffix(glob, enabledExtension) { + return nil, errors.New("Glob should have the enabledExtension as suffix") + } + + g := &GlobManager{ + glob: glob, + enabledExtension: enabledExtension, + disabledExtension: disabledExtension, + } + if err := g.load(); err != nil { + return nil, err + } + return g, nil +} + +func (g *GlobManager) load() error { + // empty previous data + g.files = nil + + // Load enabled + watcher := NewGlobWatcher(g.glob) + files, _, err := watcher.Scan() + if err != nil { + return err + } + + for _, path := range files { + // Trim cfg file name + g.files = append(g.files, &cfgfile{ + name: strings.TrimSuffix(filepath.Base(path), g.enabledExtension), + enabled: true, + path: path, + }) + } + + // Load disabled + watcher = NewGlobWatcher(g.glob + g.disabledExtension) + files, _, err = watcher.Scan() + if err != nil { + return err + } + + for _, path := range files { + // Trim cfg file name + g.files = append(g.files, &cfgfile{ + name: strings.TrimSuffix(filepath.Base(path), g.enabledExtension+g.disabledExtension), + enabled: false, + path: path, + }) + } + + return nil +} + +// ListEnabled conf files +func (g *GlobManager) ListEnabled() []string { + var names []string + for _, file := range g.files { + if file.enabled { + names = append(names, file.name) + } + } + + return names +} + +// ListDisabled conf files +func (g *GlobManager) ListDisabled() []string { + var names []string + for _, file := range g.files { + if !file.enabled { + names = append(names, file.name) + } + } + + return names +} + +// Enabled returns true if given conf file is enabled +func (g *GlobManager) Enabled(name string) bool { + for _, file := range g.files { + if name == file.name { + return file.enabled + } + } + return false +} + +// Exists return true if the given conf exists (enabled or disabled) +func (g *GlobManager) Exists(name string) bool { + for _, file := range g.files { + if name == file.name { + return true + } + } + return false +} + +// Enable given conf file, does nothing if it's enabled already +func (g *GlobManager) Enable(name string) error { + for _, file := range g.files { + if name == file.name { + if !file.enabled { + newPath := strings.TrimSuffix(file.path, g.disabledExtension) + if err := os.Rename(file.path, newPath); err != nil { + return errors.Wrap(err, "enable failed") + } + file.enabled = true + file.path = newPath + } + return nil + } + } + + return errors.Errorf("module %s not found", name) +} + +// Disable given conf file, does nothing if it's disabled already +func (g *GlobManager) Disable(name string) error { + for _, file := range g.files { + if name == file.name { + if file.enabled { + newPath := file.path + g.disabledExtension + if err := os.Rename(file.path, newPath); err != nil { + return errors.Wrap(err, "disable failed") + } + file.enabled = false + file.path = newPath + } + return nil + } + } + + return errors.Errorf("module %s not found", name) +} diff --git a/libbeat/cfgfile/glob_manager_test.go b/libbeat/cfgfile/glob_manager_test.go new file mode 100644 index 000000000000..c651986d14f1 --- /dev/null +++ b/libbeat/cfgfile/glob_manager_test.go @@ -0,0 +1,86 @@ +package cfgfile + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGlobManagerInit(t *testing.T) { + // Wrong settings return error + manager, err := NewGlobManager("dir/*.yml", ".noyml", ".disabled") + assert.Error(t, err) + assert.Nil(t, manager) +} + +func TestGlobManager(t *testing.T) { + // Create random temp directory + dir, err := ioutil.TempDir("", "glob_manager") + defer os.RemoveAll(dir) + if err != nil { + t.Fatal(err) + } + + // Prepare scenario: + content := []byte("test\n") + err = ioutil.WriteFile(dir+"/config1.yml", content, 0644) + assert.NoError(t, err) + err = ioutil.WriteFile(dir+"/config2.yml", content, 0644) + assert.NoError(t, err) + err = ioutil.WriteFile(dir+"/config3.yml.disabled", content, 0644) + assert.NoError(t, err) + + // Init Glob Manager + glob := dir + "/*.yml" + manager, err := NewGlobManager(glob, ".yml", ".disabled") + if err != nil { + t.Fatal(err) + } + + assert.True(t, manager.Exists("config1")) + assert.True(t, manager.Exists("config2")) + assert.True(t, manager.Exists("config3")) + assert.False(t, manager.Exists("config4")) + + assert.True(t, manager.Enabled("config1")) + assert.True(t, manager.Enabled("config2")) + assert.False(t, manager.Enabled("config3")) + + assert.Equal(t, len(manager.ListEnabled()), 2) + assert.Equal(t, len(manager.ListDisabled()), 1) + + // Test disable + if err = manager.Disable("config2"); err != nil { + t.Fatal(err) + } + + assert.Equal(t, len(manager.ListEnabled()), 1) + assert.Equal(t, len(manager.ListDisabled()), 2) + + assert.Equal(t, manager.ListEnabled(), []string{"config1"}) + + // Test enable + if err = manager.Enable("config3"); err != nil { + t.Fatal(err) + } + + assert.Equal(t, len(manager.ListEnabled()), 2) + assert.Equal(t, len(manager.ListDisabled()), 1) + + assert.Equal(t, manager.ListDisabled(), []string{"config2"}) + + // Check correct files layout: + files, err := filepath.Glob(dir + "/*") + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, files, []string{ + filepath.Join(dir, "config1.yml"), + filepath.Join(dir, "config2.yml.disabled"), + filepath.Join(dir, "config3.yml"), + }) +} diff --git a/libbeat/cfgfile/reload.go b/libbeat/cfgfile/reload.go index 79cf7afed1f5..e382b33c7c8a 100644 --- a/libbeat/cfgfile/reload.go +++ b/libbeat/cfgfile/reload.go @@ -12,7 +12,7 @@ import ( ) var ( - DefaultReloadConfig = ReloadConfig{ + DefaultDynamicConfig = DynamicConfig{ Reload: Reload{ Period: 10 * time.Second, Enabled: false, @@ -27,7 +27,9 @@ var ( moduleRunning = monitoring.NewInt(nil, "libbeat.config.module.running") ) -type ReloadConfig struct { +// DynamicConfig loads config files from a given path, allowing to reload new changes +// while running the beat +type DynamicConfig struct { // If path is a relative path, it is relative to the ${path.config} Path string `config:"path"` Reload Reload `config:"reload"` @@ -51,7 +53,7 @@ type Runner interface { // Reloader is used to register and reload modules type Reloader struct { registry *Registry - config ReloadConfig + config DynamicConfig done chan struct{} wg sync.WaitGroup } @@ -59,7 +61,7 @@ type Reloader struct { // NewReloader creates new Reloader instance for the given config func NewReloader(cfg *common.Config) *Reloader { - config := DefaultReloadConfig + config := DefaultDynamicConfig cfg.Unpack(&config) return &Reloader{ diff --git a/libbeat/cmd/modules.go b/libbeat/cmd/modules.go new file mode 100644 index 000000000000..c75c10954aa7 --- /dev/null +++ b/libbeat/cmd/modules.go @@ -0,0 +1,139 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/libbeat/beat" +) + +// ModulesManager interface provides all actions needed to implement modules command +// (to list, enable & disable modules) +type ModulesManager interface { + ListEnabled() []string + ListDisabled() []string + Exists(name string) bool + Enabled(name string) bool + Enable(name string) error + Disable(name string) error +} + +// ModulesManagerFactory builds and return a ModulesManager for the given Beat +type ModulesManagerFactory func(beat *beat.Beat) (ModulesManager, error) + +// GenModulesCmd initializes a command to manage a modules.d folder, it offers +// list, enable and siable actions +func GenModulesCmd(name, version string, modulesFactory ModulesManagerFactory) *cobra.Command { + modulesCmd := cobra.Command{ + Use: "modules", + Short: "Manage configured modules", + } + + modulesCmd.AddCommand(genListModulesCmd(name, version, modulesFactory)) + modulesCmd.AddCommand(genEnableModulesCmd(name, version, modulesFactory)) + modulesCmd.AddCommand(genDisableModulesCmd(name, version, modulesFactory)) + + return &modulesCmd +} + +// Instantiate a modules manager or die trying +func getModules(name, version string, modulesFactory ModulesManagerFactory) ModulesManager { + b, err := beat.New(name, version) + if err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) + os.Exit(1) + } + + if err = b.Init(); err != nil { + fmt.Fprintf(os.Stderr, "Error initializing beat: %s\n", err) + os.Exit(1) + } + + manager, err := modulesFactory(b) + if err != nil { + fmt.Fprintf(os.Stderr, "Error in modules manager: %s\n", err) + os.Exit(1) + } + + return manager +} + +func genListModulesCmd(name, version string, modulesFactory ModulesManagerFactory) *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "List modules", + Run: func(cmd *cobra.Command, args []string) { + modules := getModules(name, version, modulesFactory) + + fmt.Println("Enabled:") + for _, module := range modules.ListEnabled() { + fmt.Println(module) + } + + fmt.Println("\nDisabled:") + for _, module := range modules.ListDisabled() { + fmt.Println(module) + } + }, + } +} + +func genEnableModulesCmd(name, version string, modulesFactory ModulesManagerFactory) *cobra.Command { + return &cobra.Command{ + Use: "enable MODULE...", + Short: "Enable one or more given modules", + Run: func(cmd *cobra.Command, args []string) { + modules := getModules(name, version, modulesFactory) + + for _, module := range args { + if !modules.Exists(module) { + fmt.Printf("Module %s doesn't exists!\n", module) + os.Exit(1) + } + + if modules.Enabled(module) { + fmt.Printf("Module %s is already enabled\n", module) + continue + } + + if err := modules.Enable(module); err != nil { + fmt.Fprintf(os.Stderr, "There was an error enabling module %s: %s\n", module, err) + os.Exit(1) + } + + fmt.Printf("Enabled %s\n", module) + } + }, + } +} + +func genDisableModulesCmd(name, version string, modulesFactory ModulesManagerFactory) *cobra.Command { + return &cobra.Command{ + Use: "disable MODULE...", + Short: "Disable one or more given modules", + Run: func(cmd *cobra.Command, args []string) { + modules := getModules(name, version, modulesFactory) + + for _, module := range args { + if !modules.Exists(module) { + fmt.Fprintf(os.Stderr, "Module %s doesn't exists!\n", module) + os.Exit(1) + } + + if !modules.Enabled(module) { + fmt.Fprintf(os.Stderr, "Module %s is already disabled\n", module) + continue + } + + if err := modules.Disable(module); err != nil { + fmt.Fprintf(os.Stderr, "There was an error disabling module %s: %s\n", module, err) + os.Exit(1) + } + + fmt.Printf("Disabled %s\n", module) + } + }, + } +} diff --git a/metricbeat/Makefile b/metricbeat/Makefile index e38822a820f9..cc828aca35be 100644 --- a/metricbeat/Makefile +++ b/metricbeat/Makefile @@ -47,10 +47,14 @@ collect-docs: python-env .PHONY: configs configs: python-env @cp ${ES_BEATS}/metricbeat/_meta/common.yml _meta/beat.yml - @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml @cat ${ES_BEATS}/metricbeat/_meta/setup.yml >> _meta/beat.yml - @cp ${ES_BEATS}/metricbeat/_meta/common.reference.yml _meta/beat.reference.yml + @cat ${ES_BEATS}/metricbeat/_meta/common.reference.yml > _meta/beat.reference.yml @${PYTHON_ENV}/bin/python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.reference.yml + @rm -rf modules.d && mkdir -p modules.d + @for MODULE in `ls module | grep -v .go`; do cp -a $(PWD)/module/$$MODULE/_meta/config.yml modules.d/$$MODULE.yml.disabled; done + @chmod go-w modules.d/* + @# Enable system by default: + @if [ -f modules.d/system.yml.disabled ]; then mv modules.d/system.yml.disabled modules.d/system.yml; fi # Generates imports for all modules and metricsets .PHONY: imports @@ -63,8 +67,8 @@ imports: python-env before-build: ifeq ($(BEAT_NAME), metricbeat) # disable the system/load metricset on windows - sed -i.bk 's/- load/#- load/' $(PREFIX)/metricbeat-win.yml - rm $(PREFIX)/metricbeat-win.yml.bk + sed -i.bk 's/- load/#- load/' $(PREFIX)/modules.d-win/system.yml + rm $(PREFIX)/modules.d-win/system.yml.bk sed -i.bk 's/- load/#- load/' $(PREFIX)/metricbeat-win.reference.yml rm $(PREFIX)/metricbeat-win.reference.yml.bk endif diff --git a/metricbeat/_meta/setup.yml b/metricbeat/_meta/setup.yml index c18143a7cd42..0f8331622837 100644 --- a/metricbeat/_meta/setup.yml +++ b/metricbeat/_meta/setup.yml @@ -1,3 +1,15 @@ + +#========================== Modules configuration ============================ +metricbeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + #==================== Elasticsearch template setting ========================== setup.template.settings: index.number_of_shards: 1 diff --git a/metricbeat/beater/config.go b/metricbeat/beater/config.go index 6e7e06ac6321..7c2f9f3e9d36 100644 --- a/metricbeat/beater/config.go +++ b/metricbeat/beater/config.go @@ -10,7 +10,7 @@ import ( type Config struct { // Modules is a list of module specific configuration data. Modules []*common.Config `config:"modules"` - ReloadModules *common.Config `config:"config.modules"` + ConfigModules *common.Config `config:"config.modules"` MaxStartDelay time.Duration `config:"max_start_delay"` // Upper bound on the random startup delay for metricsets (use 0 to disable startup delay). } diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index d87e82abf28e..4140f8cb04e6 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -38,7 +38,7 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) { modules, err := module.NewWrappers(config.MaxStartDelay, config.Modules, mb.Registry) if err != nil { // Empty config is fine if dynamic config is enabled - if !config.ReloadModules.Enabled() { + if !config.ConfigModules.Enabled() { return nil, err } else if err != mb.ErrEmptyConfig && err != mb.ErrAllModulesDisabled { return nil, err @@ -72,9 +72,8 @@ func (bt *Metricbeat) Run(b *beat.Beat) error { }() } - if bt.config.ReloadModules.Enabled() { - logp.Beta("feature dynamic configuration reloading is enabled.") - moduleReloader := cfgfile.NewReloader(bt.config.ReloadModules) + if bt.config.ConfigModules.Enabled() { + moduleReloader := cfgfile.NewReloader(bt.config.ConfigModules) factory := module.NewFactory(bt.config.MaxStartDelay, b.Publisher) go moduleReloader.Run(factory) diff --git a/metricbeat/cmd/modules.go b/metricbeat/cmd/modules.go new file mode 100644 index 000000000000..f7a467e51543 --- /dev/null +++ b/metricbeat/cmd/modules.go @@ -0,0 +1,33 @@ +package cmd + +import ( + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/cfgfile" + "github.com/elastic/beats/libbeat/cmd" +) + +func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) { + config, err := beat.BeatConfig() + if err != nil { + return nil, errors.Wrap(err, "initialization error") + } + + glob, err := config.String("config.modules.path", -1) + if err != nil { + return nil, errors.Errorf("modules management requires 'metricbeat.config.modules.path' setting") + } + + if !strings.HasSuffix(glob, "*.yml") { + return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob) + } + + modulesManager, err := cfgfile.NewGlobManager(glob, ".yml", ".disabled") + if err != nil { + return nil, errors.Wrap(err, "initialization error") + } + return modulesManager, nil +} diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index bb01846fb1e1..0d81f744bc73 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -23,4 +23,5 @@ func init() { runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) RootCmd = cmd.GenRootCmdWithRunFlags(Name, "", beater.New, runFlags) + RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager)) } diff --git a/metricbeat/docs/modules/apache.asciidoc b/metricbeat/docs/modules/apache.asciidoc index b9ca1e094b54..b22ac468e35b 100644 --- a/metricbeat/docs/modules/apache.asciidoc +++ b/metricbeat/docs/modules/apache.asciidoc @@ -34,7 +34,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: apache metricsets: ["status"] - enabled: false period: 10s # Apache hosts diff --git a/metricbeat/docs/modules/ceph.asciidoc b/metricbeat/docs/modules/ceph.asciidoc index af41881c1dd4..6606fc8d8474 100644 --- a/metricbeat/docs/modules/ceph.asciidoc +++ b/metricbeat/docs/modules/ceph.asciidoc @@ -22,7 +22,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"] - enabled: false period: 10s hosts: ["localhost:5000"] ---- diff --git a/metricbeat/docs/modules/couchbase.asciidoc b/metricbeat/docs/modules/couchbase.asciidoc index e6a231b398e7..cc98537b5685 100644 --- a/metricbeat/docs/modules/couchbase.asciidoc +++ b/metricbeat/docs/modules/couchbase.asciidoc @@ -22,7 +22,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: couchbase metricsets: ["bucket", "cluster", "node"] - enabled: false period: 10s hosts: ["localhost:8091"] ---- diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc index 8ac346d36856..4a9b7e33800b 100644 --- a/metricbeat/docs/modules/docker.asciidoc +++ b/metricbeat/docs/modules/docker.asciidoc @@ -33,7 +33,6 @@ metricbeat.modules: - module: docker metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"] hosts: ["unix:///var/run/docker.sock"] - enabled: false period: 10s # To connect to Docker over TLS you must specify a client and CA certificate. diff --git a/metricbeat/docs/modules/dropwizard.asciidoc b/metricbeat/docs/modules/dropwizard.asciidoc index 970dbd812065..91ef03032ae0 100644 --- a/metricbeat/docs/modules/dropwizard.asciidoc +++ b/metricbeat/docs/modules/dropwizard.asciidoc @@ -20,7 +20,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: dropwizard metricsets: ["collector"] - enabled: false period: 10s hosts: ["localhost:8080"] metrics_path: /metrics/metrics diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc index 3de0f509783a..0ae9c1be33cd 100644 --- a/metricbeat/docs/modules/elasticsearch.asciidoc +++ b/metricbeat/docs/modules/elasticsearch.asciidoc @@ -19,11 +19,8 @@ in <>. Here is an example configuration: metricbeat.modules: - module: elasticsearch metricsets: ["node", "node_stats"] - enabled: false period: 10s hosts: ["localhost:9200"] - - ---- [float] diff --git a/metricbeat/docs/modules/golang.asciidoc b/metricbeat/docs/modules/golang.asciidoc index 2a6d741fd006..57663c5a0cf4 100644 --- a/metricbeat/docs/modules/golang.asciidoc +++ b/metricbeat/docs/modules/golang.asciidoc @@ -19,7 +19,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: golang metricsets: ["expvar","heap"] - enabled: false period: 10s hosts: ["localhost:6060"] heap.path: "/debug/vars" diff --git a/metricbeat/docs/modules/haproxy.asciidoc b/metricbeat/docs/modules/haproxy.asciidoc index 3ed714f7323f..b45c528f88c5 100644 --- a/metricbeat/docs/modules/haproxy.asciidoc +++ b/metricbeat/docs/modules/haproxy.asciidoc @@ -32,7 +32,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: haproxy metricsets: ["info", "stat"] - enabled: false period: 10s hosts: ["tcp://127.0.0.1:14567"] ---- diff --git a/metricbeat/docs/modules/http.asciidoc b/metricbeat/docs/modules/http.asciidoc index 86d929ba07d6..3de4f5da9097 100644 --- a/metricbeat/docs/modules/http.asciidoc +++ b/metricbeat/docs/modules/http.asciidoc @@ -25,7 +25,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: http metricsets: ["json"] - enabled: false period: 10s hosts: ["localhost:80"] namespace: "json_namespace" diff --git a/metricbeat/docs/modules/jolokia.asciidoc b/metricbeat/docs/modules/jolokia.asciidoc index f68ff302f5d8..10ade8625fdf 100644 --- a/metricbeat/docs/modules/jolokia.asciidoc +++ b/metricbeat/docs/modules/jolokia.asciidoc @@ -22,7 +22,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: jolokia metricsets: ["jmx"] - enabled: false period: 10s hosts: ["localhost"] namespace: "metrics" diff --git a/metricbeat/docs/modules/kafka.asciidoc b/metricbeat/docs/modules/kafka.asciidoc index 058a3b444a7b..6541affa2370 100644 --- a/metricbeat/docs/modules/kafka.asciidoc +++ b/metricbeat/docs/modules/kafka.asciidoc @@ -22,7 +22,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: kafka metricsets: ["partition"] - enabled: false period: 10s hosts: ["localhost:9092"] diff --git a/metricbeat/docs/modules/kibana.asciidoc b/metricbeat/docs/modules/kibana.asciidoc index ff90a453ce21..ee6e8f826191 100644 --- a/metricbeat/docs/modules/kibana.asciidoc +++ b/metricbeat/docs/modules/kibana.asciidoc @@ -19,7 +19,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: kibana metricsets: ["status"] - enabled: false period: 10s hosts: ["localhost:5601"] ---- diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index de3a789f9c48..f142f015204a 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -26,7 +26,6 @@ in <>. Here is an example configuration: metricbeat.modules: # Node metrics, from kubelet: - module: kubernetes - enabled: false metricsets: - node - system @@ -53,7 +52,6 @@ metricbeat.modules: enabled: false metricsets: - event - in_cluster: true ---- [float] diff --git a/metricbeat/docs/modules/memcached.asciidoc b/metricbeat/docs/modules/memcached.asciidoc index b24dd5b9c7d6..cf17cb99626c 100644 --- a/metricbeat/docs/modules/memcached.asciidoc +++ b/metricbeat/docs/modules/memcached.asciidoc @@ -20,10 +20,8 @@ in <>. Here is an example configuration: metricbeat.modules: - module: memcached metricsets: ["stats"] - enabled: false period: 10s hosts: ["localhost:11211"] - ---- [float] diff --git a/metricbeat/docs/modules/mongodb.asciidoc b/metricbeat/docs/modules/mongodb.asciidoc index 7d625416ceb5..c6c5cd9293c4 100644 --- a/metricbeat/docs/modules/mongodb.asciidoc +++ b/metricbeat/docs/modules/mongodb.asciidoc @@ -65,7 +65,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: mongodb metricsets: ["dbstats", "status"] - enabled: false period: 10s # The hosts must be passed as MongoDB URLs in the format: diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc index e84702b1863f..39c2f124f9ea 100644 --- a/metricbeat/docs/modules/mysql.asciidoc +++ b/metricbeat/docs/modules/mysql.asciidoc @@ -48,7 +48,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: mysql metricsets: ["status"] - enabled: false period: 10s # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" diff --git a/metricbeat/docs/modules/nginx.asciidoc b/metricbeat/docs/modules/nginx.asciidoc index 1c280f4386aa..f240267f1ad3 100644 --- a/metricbeat/docs/modules/nginx.asciidoc +++ b/metricbeat/docs/modules/nginx.asciidoc @@ -26,7 +26,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: nginx metricsets: ["stubstatus"] - enabled: false period: 10s # Nginx hosts diff --git a/metricbeat/docs/modules/php_fpm.asciidoc b/metricbeat/docs/modules/php_fpm.asciidoc index 40fce8314e01..5ece5e824723 100644 --- a/metricbeat/docs/modules/php_fpm.asciidoc +++ b/metricbeat/docs/modules/php_fpm.asciidoc @@ -46,7 +46,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: php_fpm metricsets: ["pool"] - enabled: false period: 10s status_path: "/status" hosts: ["localhost:8080"] diff --git a/metricbeat/docs/modules/postgresql.asciidoc b/metricbeat/docs/modules/postgresql.asciidoc index 1c8876ff04c8..7adc4f2cc27a 100644 --- a/metricbeat/docs/modules/postgresql.asciidoc +++ b/metricbeat/docs/modules/postgresql.asciidoc @@ -74,7 +74,6 @@ metricbeat.modules: # Stats about every PostgreSQL process - activity - enabled: false period: 10s # The host must be passed as PostgreSQL URL. Example: @@ -88,7 +87,6 @@ metricbeat.modules: # Password to use when connecting to PostgreSQL. Empty by default. #password: pass - ---- [float] diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index b25d9256a824..d84e2dc5f651 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -22,7 +22,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: prometheus metricsets: ["stats"] - enabled: false period: 10s hosts: ["localhost:9090"] metrics_path: /metrics diff --git a/metricbeat/docs/modules/rabbitmq.asciidoc b/metricbeat/docs/modules/rabbitmq.asciidoc index da2db7b84283..4eb9c1617d8c 100644 --- a/metricbeat/docs/modules/rabbitmq.asciidoc +++ b/metricbeat/docs/modules/rabbitmq.asciidoc @@ -20,7 +20,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: rabbitmq metricsets: ["node"] - enabled: false period: 10s hosts: ["localhost:15672"] diff --git a/metricbeat/docs/modules/redis.asciidoc b/metricbeat/docs/modules/redis.asciidoc index 5c3923d4c0a4..8cbd57ce4067 100644 --- a/metricbeat/docs/modules/redis.asciidoc +++ b/metricbeat/docs/modules/redis.asciidoc @@ -42,7 +42,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: redis metricsets: ["info", "keyspace"] - enabled: false period: 10s # Redis hosts diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index db0c104558fc..e9ae39dae7c1 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -86,7 +86,6 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: system - enabled: true period: 10s metricsets: - cpu @@ -102,8 +101,8 @@ metricbeat.modules: process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory + - module: system - enabled: true period: 1m metricsets: - filesystem diff --git a/metricbeat/docs/modules/vsphere.asciidoc b/metricbeat/docs/modules/vsphere.asciidoc index e3ae92d6b8bd..3ecec200c2a5 100644 --- a/metricbeat/docs/modules/vsphere.asciidoc +++ b/metricbeat/docs/modules/vsphere.asciidoc @@ -20,7 +20,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: vsphere metricsets: ["datastore, host, virtualmachine"] - enabled: false period: 10s hosts: ["https://localhost/sdk"] diff --git a/metricbeat/docs/modules/windows.asciidoc b/metricbeat/docs/modules/windows.asciidoc index 97869af3f978..ec923ac71ede 100644 --- a/metricbeat/docs/modules/windows.asciidoc +++ b/metricbeat/docs/modules/windows.asciidoc @@ -19,7 +19,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: windows metricsets: ["perfmon"] - enabled: false period: 10s perfmon.counters: ---- diff --git a/metricbeat/docs/modules/zookeeper.asciidoc b/metricbeat/docs/modules/zookeeper.asciidoc index 8d861ea52d2c..733240437e20 100644 --- a/metricbeat/docs/modules/zookeeper.asciidoc +++ b/metricbeat/docs/modules/zookeeper.asciidoc @@ -25,7 +25,6 @@ in <>. Here is an example configuration: metricbeat.modules: - module: zookeeper metricsets: ["mntr"] - enabled: false period: 10s hosts: ["localhost:2181"] ---- diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 8707aed301dc..3bd1a8ad6e36 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -93,7 +93,6 @@ metricbeat.modules: #------------------------------- Apache Module ------------------------------- - module: apache metricsets: ["status"] - enabled: false period: 10s # Apache hosts @@ -111,14 +110,12 @@ metricbeat.modules: #-------------------------------- ceph Module -------------------------------- - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"] - enabled: false period: 10s hosts: ["localhost:5000"] #------------------------------ Couchbase Module ----------------------------- - module: couchbase metricsets: ["bucket", "cluster", "node"] - enabled: false period: 10s hosts: ["localhost:8091"] @@ -126,7 +123,6 @@ metricbeat.modules: - module: docker metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"] hosts: ["unix:///var/run/docker.sock"] - enabled: false period: 10s # To connect to Docker over TLS you must specify a client and CA certificate. @@ -138,7 +134,6 @@ metricbeat.modules: #----------------------------- Dropwizard Module ----------------------------- - module: dropwizard metricsets: ["collector"] - enabled: false period: 10s hosts: ["localhost:8080"] metrics_path: /metrics/metrics @@ -147,16 +142,12 @@ metricbeat.modules: #---------------------------- elasticsearch Module --------------------------- - module: elasticsearch metricsets: ["node", "node_stats"] - enabled: false period: 10s hosts: ["localhost:9200"] - - #------------------------------- golang Module ------------------------------- - module: golang metricsets: ["expvar","heap"] - enabled: false period: 10s hosts: ["localhost:6060"] heap.path: "/debug/vars" @@ -167,14 +158,12 @@ metricbeat.modules: #------------------------------- HAProxy Module ------------------------------ - module: haproxy metricsets: ["info", "stat"] - enabled: false period: 10s hosts: ["tcp://127.0.0.1:14567"] #-------------------------------- HTTP Module -------------------------------- - module: http metricsets: ["json"] - enabled: false period: 10s hosts: ["localhost:80"] namespace: "json_namespace" @@ -187,7 +176,6 @@ metricbeat.modules: #------------------------------- Jolokia Module ------------------------------ - module: jolokia metricsets: ["jmx"] - enabled: false period: 10s hosts: ["localhost"] namespace: "metrics" @@ -199,7 +187,6 @@ metricbeat.modules: #-------------------------------- kafka Module ------------------------------- - module: kafka metricsets: ["partition"] - enabled: false period: 10s hosts: ["localhost:9092"] @@ -227,14 +214,12 @@ metricbeat.modules: #------------------------------- kibana Module ------------------------------- - module: kibana metricsets: ["status"] - enabled: false period: 10s hosts: ["localhost:5601"] #----------------------------- kubernetes Module ----------------------------- # Node metrics, from kubelet: - module: kubernetes - enabled: false metricsets: - node - system @@ -261,20 +246,16 @@ metricbeat.modules: enabled: false metricsets: - event - in_cluster: true #------------------------------ memcached Module ----------------------------- - module: memcached metricsets: ["stats"] - enabled: false period: 10s hosts: ["localhost:11211"] - #------------------------------- MongoDB Module ------------------------------ - module: mongodb metricsets: ["dbstats", "status"] - enabled: false period: 10s # The hosts must be passed as MongoDB URLs in the format: @@ -293,7 +274,6 @@ metricbeat.modules: #-------------------------------- MySQL Module ------------------------------- - module: mysql metricsets: ["status"] - enabled: false period: 10s # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" @@ -325,7 +305,6 @@ metricbeat.modules: #------------------------------- php_fpm Module ------------------------------ - module: php_fpm metricsets: ["pool"] - enabled: false period: 10s status_path: "/status" hosts: ["localhost:8080"] @@ -342,7 +321,6 @@ metricbeat.modules: # Stats about every PostgreSQL process - activity - enabled: false period: 10s # The host must be passed as PostgreSQL URL. Example: @@ -357,11 +335,9 @@ metricbeat.modules: # Password to use when connecting to PostgreSQL. Empty by default. #password: pass - #----------------------------- Prometheus Module ----------------------------- - module: prometheus metricsets: ["stats"] - enabled: false period: 10s hosts: ["localhost:9090"] metrics_path: /metrics @@ -370,7 +346,6 @@ metricbeat.modules: #------------------------------ rabbitmq Module ------------------------------ - module: rabbitmq metricsets: ["node"] - enabled: false period: 10s hosts: ["localhost:15672"] @@ -380,7 +355,6 @@ metricbeat.modules: #-------------------------------- Redis Module ------------------------------- - module: redis metricsets: ["info", "keyspace"] - enabled: false period: 10s # Redis hosts @@ -412,7 +386,6 @@ metricbeat.modules: #------------------------------- vsphere Module ------------------------------ - module: vsphere metricsets: ["datastore, host, virtualmachine"] - enabled: false period: 10s hosts: ["https://localhost/sdk"] @@ -424,14 +397,12 @@ metricbeat.modules: #------------------------------- Windows Module ------------------------------ - module: windows metricsets: ["perfmon"] - enabled: false period: 10s perfmon.counters: #------------------------------ ZooKeeper Module ----------------------------- - module: zookeeper metricsets: ["mntr"] - enabled: false period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 00d3aa5500ed..632432709983 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -8,35 +8,15 @@ # https://www.elastic.co/guide/en/beats/metricbeat/index.html #========================== Modules configuration ============================ -metricbeat.modules: - -#------------------------------- System Module ------------------------------- -- module: system - enabled: true - period: 10s - metricsets: - - cpu - - load - - memory - - network - - process - - process_summary - #- core - #- diskio - #- socket - processes: ['.*'] - process.include_top_n: - by_cpu: 5 # include top 5 processes by CPU - by_memory: 5 # include top 5 processes by memory -- module: system - enabled: true - period: 1m - metricsets: - - filesystem - - fsstat - filters: - - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' +metricbeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s #==================== Elasticsearch template setting ========================== setup.template.settings: diff --git a/metricbeat/module/apache/_meta/config.reference.yml b/metricbeat/module/apache/_meta/config.reference.yml index 40a9ccfe14ce..74f4266334f0 100644 --- a/metricbeat/module/apache/_meta/config.reference.yml +++ b/metricbeat/module/apache/_meta/config.reference.yml @@ -1,6 +1,5 @@ - module: apache metricsets: ["status"] - enabled: false period: 10s # Apache hosts diff --git a/metricbeat/module/apache/_meta/config.yml b/metricbeat/module/apache/_meta/config.yml index 72ea8610b7e3..08d1efaf92c5 100644 --- a/metricbeat/module/apache/_meta/config.yml +++ b/metricbeat/module/apache/_meta/config.yml @@ -1,6 +1,5 @@ - module: apache metricsets: ["status"] - enabled: false period: 10s # Apache hosts diff --git a/metricbeat/module/ceph/_meta/config.yml b/metricbeat/module/ceph/_meta/config.yml index c8ae607c4a04..21ee68217332 100644 --- a/metricbeat/module/ceph/_meta/config.yml +++ b/metricbeat/module/ceph/_meta/config.yml @@ -1,5 +1,4 @@ - module: ceph metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"] - enabled: false period: 10s hosts: ["localhost:5000"] diff --git a/metricbeat/module/couchbase/_meta/config.yml b/metricbeat/module/couchbase/_meta/config.yml index dda1bb4731b4..2691d77a1b4a 100644 --- a/metricbeat/module/couchbase/_meta/config.yml +++ b/metricbeat/module/couchbase/_meta/config.yml @@ -1,5 +1,4 @@ - module: couchbase metricsets: ["bucket", "cluster", "node"] - enabled: false period: 10s hosts: ["localhost:8091"] diff --git a/metricbeat/module/docker/_meta/config.yml b/metricbeat/module/docker/_meta/config.yml index 809b867afa35..870492584014 100644 --- a/metricbeat/module/docker/_meta/config.yml +++ b/metricbeat/module/docker/_meta/config.yml @@ -1,7 +1,6 @@ - module: docker metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"] hosts: ["unix:///var/run/docker.sock"] - enabled: false period: 10s # To connect to Docker over TLS you must specify a client and CA certificate. diff --git a/metricbeat/module/dropwizard/_meta/config.yml b/metricbeat/module/dropwizard/_meta/config.yml index adf7ff28f10a..6dfa3755ca0b 100644 --- a/metricbeat/module/dropwizard/_meta/config.yml +++ b/metricbeat/module/dropwizard/_meta/config.yml @@ -1,6 +1,5 @@ - module: dropwizard metricsets: ["collector"] - enabled: false period: 10s hosts: ["localhost:8080"] metrics_path: /metrics/metrics diff --git a/metricbeat/module/elasticsearch/_meta/config.yml b/metricbeat/module/elasticsearch/_meta/config.yml index 443979ea5498..a792d2bee9de 100644 --- a/metricbeat/module/elasticsearch/_meta/config.yml +++ b/metricbeat/module/elasticsearch/_meta/config.yml @@ -1,7 +1,4 @@ - module: elasticsearch metricsets: ["node", "node_stats"] - enabled: false period: 10s hosts: ["localhost:9200"] - - diff --git a/metricbeat/module/golang/_meta/config.yml b/metricbeat/module/golang/_meta/config.yml index 440b9e780555..ba9db8e10899 100644 --- a/metricbeat/module/golang/_meta/config.yml +++ b/metricbeat/module/golang/_meta/config.yml @@ -1,6 +1,5 @@ - module: golang metricsets: ["expvar","heap"] - enabled: false period: 10s hosts: ["localhost:6060"] heap.path: "/debug/vars" diff --git a/metricbeat/module/haproxy/_meta/config.yml b/metricbeat/module/haproxy/_meta/config.yml index fcee8d3b8fe9..febce8d22468 100644 --- a/metricbeat/module/haproxy/_meta/config.yml +++ b/metricbeat/module/haproxy/_meta/config.yml @@ -1,5 +1,4 @@ - module: haproxy metricsets: ["info", "stat"] - enabled: false period: 10s hosts: ["tcp://127.0.0.1:14567"] diff --git a/metricbeat/module/http/_meta/config.yml b/metricbeat/module/http/_meta/config.yml index 936c435b964a..aa40cfb7cc1e 100644 --- a/metricbeat/module/http/_meta/config.yml +++ b/metricbeat/module/http/_meta/config.yml @@ -1,6 +1,5 @@ - module: http metricsets: ["json"] - enabled: false period: 10s hosts: ["localhost:80"] namespace: "json_namespace" diff --git a/metricbeat/module/jolokia/_meta/config.yml b/metricbeat/module/jolokia/_meta/config.yml index b7bd87fd3ad4..de331606b85e 100644 --- a/metricbeat/module/jolokia/_meta/config.yml +++ b/metricbeat/module/jolokia/_meta/config.yml @@ -1,6 +1,5 @@ - module: jolokia metricsets: ["jmx"] - enabled: false period: 10s hosts: ["localhost"] namespace: "metrics" diff --git a/metricbeat/module/kafka/_meta/config.yml b/metricbeat/module/kafka/_meta/config.yml index f569f71d7d80..91ce183a0f12 100644 --- a/metricbeat/module/kafka/_meta/config.yml +++ b/metricbeat/module/kafka/_meta/config.yml @@ -1,6 +1,5 @@ - module: kafka metricsets: ["partition"] - enabled: false period: 10s hosts: ["localhost:9092"] diff --git a/metricbeat/module/kibana/_meta/config.yml b/metricbeat/module/kibana/_meta/config.yml index bd673be56586..09a302956797 100644 --- a/metricbeat/module/kibana/_meta/config.yml +++ b/metricbeat/module/kibana/_meta/config.yml @@ -1,5 +1,4 @@ - module: kibana metricsets: ["status"] - enabled: false period: 10s hosts: ["localhost:5601"] diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index a075952bf3a9..cf00d10cd454 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -1,6 +1,5 @@ # Node metrics, from kubelet: - module: kubernetes - enabled: false metricsets: - node - system @@ -27,4 +26,3 @@ enabled: false metricsets: - event - in_cluster: true diff --git a/metricbeat/module/memcached/_meta/config.yml b/metricbeat/module/memcached/_meta/config.yml index 85632bddb583..1b230087fab0 100644 --- a/metricbeat/module/memcached/_meta/config.yml +++ b/metricbeat/module/memcached/_meta/config.yml @@ -1,6 +1,4 @@ - module: memcached metricsets: ["stats"] - enabled: false period: 10s hosts: ["localhost:11211"] - diff --git a/metricbeat/module/mongodb/_meta/config.yml b/metricbeat/module/mongodb/_meta/config.yml index 5eb0d5994114..fb6b19e7c68f 100644 --- a/metricbeat/module/mongodb/_meta/config.yml +++ b/metricbeat/module/mongodb/_meta/config.yml @@ -1,6 +1,5 @@ - module: mongodb metricsets: ["dbstats", "status"] - enabled: false period: 10s # The hosts must be passed as MongoDB URLs in the format: diff --git a/metricbeat/module/mysql/_meta/config.yml b/metricbeat/module/mysql/_meta/config.yml index 9fa5f01116da..afff5ff7201a 100644 --- a/metricbeat/module/mysql/_meta/config.yml +++ b/metricbeat/module/mysql/_meta/config.yml @@ -1,6 +1,5 @@ - module: mysql metricsets: ["status"] - enabled: false period: 10s # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" diff --git a/metricbeat/module/nginx/_meta/config.yml b/metricbeat/module/nginx/_meta/config.yml index e9bb1c84e8ba..e3737ed986e4 100644 --- a/metricbeat/module/nginx/_meta/config.yml +++ b/metricbeat/module/nginx/_meta/config.yml @@ -1,6 +1,5 @@ - module: nginx metricsets: ["stubstatus"] - enabled: false period: 10s # Nginx hosts diff --git a/metricbeat/module/php_fpm/_meta/config.yml b/metricbeat/module/php_fpm/_meta/config.yml index e4cdf3928661..be5764514046 100644 --- a/metricbeat/module/php_fpm/_meta/config.yml +++ b/metricbeat/module/php_fpm/_meta/config.yml @@ -1,6 +1,5 @@ - module: php_fpm metricsets: ["pool"] - enabled: false period: 10s status_path: "/status" hosts: ["localhost:8080"] diff --git a/metricbeat/module/postgresql/_meta/config.yml b/metricbeat/module/postgresql/_meta/config.yml index 911b029a5c3e..383c66d9f4f8 100644 --- a/metricbeat/module/postgresql/_meta/config.yml +++ b/metricbeat/module/postgresql/_meta/config.yml @@ -9,7 +9,6 @@ # Stats about every PostgreSQL process - activity - enabled: false period: 10s # The host must be passed as PostgreSQL URL. Example: @@ -23,4 +22,3 @@ # Password to use when connecting to PostgreSQL. Empty by default. #password: pass - diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index 084dc2d00b8a..76bee3491954 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -1,6 +1,5 @@ - module: prometheus metricsets: ["stats"] - enabled: false period: 10s hosts: ["localhost:9090"] metrics_path: /metrics diff --git a/metricbeat/module/rabbitmq/_meta/config.yml b/metricbeat/module/rabbitmq/_meta/config.yml index cee569d91c27..f45427f547b8 100644 --- a/metricbeat/module/rabbitmq/_meta/config.yml +++ b/metricbeat/module/rabbitmq/_meta/config.yml @@ -1,6 +1,5 @@ - module: rabbitmq metricsets: ["node"] - enabled: false period: 10s hosts: ["localhost:15672"] diff --git a/metricbeat/module/redis/_meta/config.yml b/metricbeat/module/redis/_meta/config.yml index e9d3895a40b4..3f3e29ec5a40 100644 --- a/metricbeat/module/redis/_meta/config.yml +++ b/metricbeat/module/redis/_meta/config.yml @@ -1,6 +1,5 @@ - module: redis metricsets: ["info", "keyspace"] - enabled: false period: 10s # Redis hosts diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml index 111aa68a2eea..d994e02c2e77 100644 --- a/metricbeat/module/system/_meta/config.yml +++ b/metricbeat/module/system/_meta/config.yml @@ -1,5 +1,4 @@ - module: system - enabled: true period: 10s metricsets: - cpu @@ -15,8 +14,8 @@ process.include_top_n: by_cpu: 5 # include top 5 processes by CPU by_memory: 5 # include top 5 processes by memory + - module: system - enabled: true period: 1m metricsets: - filesystem diff --git a/metricbeat/module/vsphere/_meta/config.yml b/metricbeat/module/vsphere/_meta/config.yml index d2c73e2473d3..171e1d0224ba 100644 --- a/metricbeat/module/vsphere/_meta/config.yml +++ b/metricbeat/module/vsphere/_meta/config.yml @@ -1,6 +1,5 @@ - module: vsphere metricsets: ["datastore, host, virtualmachine"] - enabled: false period: 10s hosts: ["https://localhost/sdk"] diff --git a/metricbeat/module/windows/_meta/config.yml b/metricbeat/module/windows/_meta/config.yml index 968ba4c51343..fc93050dabfb 100644 --- a/metricbeat/module/windows/_meta/config.yml +++ b/metricbeat/module/windows/_meta/config.yml @@ -1,5 +1,4 @@ - module: windows metricsets: ["perfmon"] - enabled: false period: 10s perfmon.counters: diff --git a/metricbeat/module/zookeeper/_meta/config.yml b/metricbeat/module/zookeeper/_meta/config.yml index 77eb186097a2..63543cefdd71 100644 --- a/metricbeat/module/zookeeper/_meta/config.yml +++ b/metricbeat/module/zookeeper/_meta/config.yml @@ -1,5 +1,4 @@ - module: zookeeper metricsets: ["mntr"] - enabled: false period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/modules.d/aerospike.yml.disabled b/metricbeat/modules.d/aerospike.yml.disabled new file mode 100644 index 000000000000..787e314dc60d --- /dev/null +++ b/metricbeat/modules.d/aerospike.yml.disabled @@ -0,0 +1,5 @@ +- module: aerospike + metricsets: ["namespace"] + enabled: false + period: 10s + hosts: ["localhost:3000"] diff --git a/metricbeat/modules.d/apache.yml.disabled b/metricbeat/modules.d/apache.yml.disabled new file mode 100644 index 000000000000..08d1efaf92c5 --- /dev/null +++ b/metricbeat/modules.d/apache.yml.disabled @@ -0,0 +1,6 @@ +- module: apache + metricsets: ["status"] + period: 10s + + # Apache hosts + hosts: ["http://127.0.0.1"] diff --git a/metricbeat/modules.d/ceph.yml.disabled b/metricbeat/modules.d/ceph.yml.disabled new file mode 100644 index 000000000000..21ee68217332 --- /dev/null +++ b/metricbeat/modules.d/ceph.yml.disabled @@ -0,0 +1,4 @@ +- module: ceph + metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk"] + period: 10s + hosts: ["localhost:5000"] diff --git a/metricbeat/modules.d/couchbase.yml.disabled b/metricbeat/modules.d/couchbase.yml.disabled new file mode 100644 index 000000000000..2691d77a1b4a --- /dev/null +++ b/metricbeat/modules.d/couchbase.yml.disabled @@ -0,0 +1,4 @@ +- module: couchbase + metricsets: ["bucket", "cluster", "node"] + period: 10s + hosts: ["localhost:8091"] diff --git a/metricbeat/modules.d/docker.yml.disabled b/metricbeat/modules.d/docker.yml.disabled new file mode 100644 index 000000000000..870492584014 --- /dev/null +++ b/metricbeat/modules.d/docker.yml.disabled @@ -0,0 +1,10 @@ +- module: docker + metricsets: ["container", "cpu", "diskio", "healthcheck", "info", "memory", "network"] + hosts: ["unix:///var/run/docker.sock"] + period: 10s + + # To connect to Docker over TLS you must specify a client and CA certificate. + #ssl: + #certificate_authority: "/etc/pki/root/ca.pem" + #certificate: "/etc/pki/client/cert.pem" + #key: "/etc/pki/client/cert.key" diff --git a/metricbeat/modules.d/dropwizard.yml.disabled b/metricbeat/modules.d/dropwizard.yml.disabled new file mode 100644 index 000000000000..6dfa3755ca0b --- /dev/null +++ b/metricbeat/modules.d/dropwizard.yml.disabled @@ -0,0 +1,6 @@ +- module: dropwizard + metricsets: ["collector"] + period: 10s + hosts: ["localhost:8080"] + metrics_path: /metrics/metrics + namespace: example diff --git a/metricbeat/modules.d/elasticsearch.yml.disabled b/metricbeat/modules.d/elasticsearch.yml.disabled new file mode 100644 index 000000000000..a792d2bee9de --- /dev/null +++ b/metricbeat/modules.d/elasticsearch.yml.disabled @@ -0,0 +1,4 @@ +- module: elasticsearch + metricsets: ["node", "node_stats"] + period: 10s + hosts: ["localhost:9200"] diff --git a/metricbeat/modules.d/golang.yml.disabled b/metricbeat/modules.d/golang.yml.disabled new file mode 100644 index 000000000000..ba9db8e10899 --- /dev/null +++ b/metricbeat/modules.d/golang.yml.disabled @@ -0,0 +1,8 @@ +- module: golang + metricsets: ["expvar","heap"] + period: 10s + hosts: ["localhost:6060"] + heap.path: "/debug/vars" + expvar: + namespace: "example" + path: "/debug/vars" diff --git a/metricbeat/modules.d/haproxy.yml.disabled b/metricbeat/modules.d/haproxy.yml.disabled new file mode 100644 index 000000000000..febce8d22468 --- /dev/null +++ b/metricbeat/modules.d/haproxy.yml.disabled @@ -0,0 +1,4 @@ +- module: haproxy + metricsets: ["info", "stat"] + period: 10s + hosts: ["tcp://127.0.0.1:14567"] diff --git a/metricbeat/modules.d/http.yml.disabled b/metricbeat/modules.d/http.yml.disabled new file mode 100644 index 000000000000..aa40cfb7cc1e --- /dev/null +++ b/metricbeat/modules.d/http.yml.disabled @@ -0,0 +1,10 @@ +- module: http + metricsets: ["json"] + period: 10s + hosts: ["localhost:80"] + namespace: "json_namespace" + path: "/" + #body: "" + #method: "GET" + #request.enabled: false + #response.enabled: false diff --git a/metricbeat/modules.d/jolokia.yml.disabled b/metricbeat/modules.d/jolokia.yml.disabled new file mode 100644 index 000000000000..de331606b85e --- /dev/null +++ b/metricbeat/modules.d/jolokia.yml.disabled @@ -0,0 +1,9 @@ +- module: jolokia + metricsets: ["jmx"] + period: 10s + hosts: ["localhost"] + namespace: "metrics" + path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" + jmx.mapping: + jmx.application: + jmx.instance: diff --git a/metricbeat/modules.d/kafka.yml.disabled b/metricbeat/modules.d/kafka.yml.disabled new file mode 100644 index 000000000000..91ce183a0f12 --- /dev/null +++ b/metricbeat/modules.d/kafka.yml.disabled @@ -0,0 +1,25 @@ +- module: kafka + metricsets: ["partition"] + period: 10s + hosts: ["localhost:9092"] + + #client_id: metricbeat + #retries: 3 + #backoff: 250ms + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # SASL authentication + #username: "" + #password: "" diff --git a/metricbeat/modules.d/kibana.yml.disabled b/metricbeat/modules.d/kibana.yml.disabled new file mode 100644 index 000000000000..09a302956797 --- /dev/null +++ b/metricbeat/modules.d/kibana.yml.disabled @@ -0,0 +1,4 @@ +- module: kibana + metricsets: ["status"] + period: 10s + hosts: ["localhost:5601"] diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled new file mode 100644 index 000000000000..cf00d10cd454 --- /dev/null +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -0,0 +1,28 @@ +# Node metrics, from kubelet: +- module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + hosts: ["localhost:10255"] + +# State metrics from kube-state-metrics service: +- module: kubernetes + enabled: false + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + period: 10s + hosts: ["kube-state-metrics:8080"] + +# Kubernetes events +- module: kubernetes + enabled: false + metricsets: + - event diff --git a/metricbeat/modules.d/memcached.yml.disabled b/metricbeat/modules.d/memcached.yml.disabled new file mode 100644 index 000000000000..1b230087fab0 --- /dev/null +++ b/metricbeat/modules.d/memcached.yml.disabled @@ -0,0 +1,4 @@ +- module: memcached + metricsets: ["stats"] + period: 10s + hosts: ["localhost:11211"] diff --git a/metricbeat/modules.d/mysql.yml.disabled b/metricbeat/modules.d/mysql.yml.disabled new file mode 100644 index 000000000000..afff5ff7201a --- /dev/null +++ b/metricbeat/modules.d/mysql.yml.disabled @@ -0,0 +1,17 @@ +- module: mysql + metricsets: ["status"] + period: 10s + + # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" + # The username and password can either be set in the DSN or using the username + # and password config options. Those specified in the DSN take precedence. + hosts: ["root:secret@tcp(127.0.0.1:3306)/"] + + # Username of hosts. Empty by default. + #username: root + + # Password of hosts. Empty by default. + #password: secret + + # By setting raw to true, all raw fields from the status metricset will be added to the event. + #raw: false diff --git a/metricbeat/modules.d/nginx.yml.disabled b/metricbeat/modules.d/nginx.yml.disabled new file mode 100644 index 000000000000..e3737ed986e4 --- /dev/null +++ b/metricbeat/modules.d/nginx.yml.disabled @@ -0,0 +1,9 @@ +- module: nginx + metricsets: ["stubstatus"] + period: 10s + + # Nginx hosts + hosts: ["http://127.0.0.1"] + + # Path to server status. Default server-status + #server_status_path: "server-status" diff --git a/metricbeat/modules.d/php_fpm.yml.disabled b/metricbeat/modules.d/php_fpm.yml.disabled new file mode 100644 index 000000000000..be5764514046 --- /dev/null +++ b/metricbeat/modules.d/php_fpm.yml.disabled @@ -0,0 +1,5 @@ +- module: php_fpm + metricsets: ["pool"] + period: 10s + status_path: "/status" + hosts: ["localhost:8080"] diff --git a/metricbeat/modules.d/postgresql.yml.disabled b/metricbeat/modules.d/postgresql.yml.disabled new file mode 100644 index 000000000000..383c66d9f4f8 --- /dev/null +++ b/metricbeat/modules.d/postgresql.yml.disabled @@ -0,0 +1,24 @@ +- module: postgresql + metricsets: + # Stats about every PostgreSQL database + - database + + # Stats about the background writer process's activity + - bgwriter + + # Stats about every PostgreSQL process + - activity + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost:5432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled new file mode 100644 index 000000000000..76bee3491954 --- /dev/null +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -0,0 +1,6 @@ +- module: prometheus + metricsets: ["stats"] + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + #namespace: example diff --git a/metricbeat/modules.d/rabbitmq.yml.disabled b/metricbeat/modules.d/rabbitmq.yml.disabled new file mode 100644 index 000000000000..f45427f547b8 --- /dev/null +++ b/metricbeat/modules.d/rabbitmq.yml.disabled @@ -0,0 +1,7 @@ +- module: rabbitmq + metricsets: ["node"] + period: 10s + hosts: ["localhost:15672"] + + username: guest + password: guest diff --git a/metricbeat/modules.d/redis.yml.disabled b/metricbeat/modules.d/redis.yml.disabled new file mode 100644 index 000000000000..3f3e29ec5a40 --- /dev/null +++ b/metricbeat/modules.d/redis.yml.disabled @@ -0,0 +1,29 @@ +- module: redis + metricsets: ["info", "keyspace"] + period: 10s + + # Redis hosts + hosts: ["127.0.0.1:6379"] + + # Timeout after which time a metricset should return an error + # Timeout is by default defined as period, as a fetch of a metricset + # should never take longer then period, as otherwise calls can pile up. + #timeout: 1s + + # Optional fields to be added to each event + #fields: + # datacenter: west + + # Network type to be used for redis connection. Default: tcp + #network: tcp + + # Max number of concurrent connections. Default: 10 + #maxconn: 10 + + # Filters can be used to reduce the number of fields sent. + #filters: + # - include_fields: + # fields: ["stats"] + + # Redis AUTH password. Empty by default. + #password: foobared diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml new file mode 100644 index 000000000000..d994e02c2e77 --- /dev/null +++ b/metricbeat/modules.d/system.yml @@ -0,0 +1,24 @@ +- module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + #- core + #- diskio + #- socket + processes: ['.*'] + process.include_top_n: + by_cpu: 5 # include top 5 processes by CPU + by_memory: 5 # include top 5 processes by memory + +- module: system + period: 1m + metricsets: + - filesystem + - fsstat + filters: + - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)' diff --git a/metricbeat/modules.d/vsphere.yml.disabled b/metricbeat/modules.d/vsphere.yml.disabled new file mode 100644 index 000000000000..171e1d0224ba --- /dev/null +++ b/metricbeat/modules.d/vsphere.yml.disabled @@ -0,0 +1,9 @@ +- module: vsphere + metricsets: ["datastore, host, virtualmachine"] + period: 10s + hosts: ["https://localhost/sdk"] + + username: "user" + password: "password" + # If insecure is true, don't verify the server's certificate chain + insecure: false diff --git a/metricbeat/modules.d/windows.yml.disabled b/metricbeat/modules.d/windows.yml.disabled new file mode 100644 index 000000000000..fc93050dabfb --- /dev/null +++ b/metricbeat/modules.d/windows.yml.disabled @@ -0,0 +1,4 @@ +- module: windows + metricsets: ["perfmon"] + period: 10s + perfmon.counters: diff --git a/metricbeat/modules.d/zookeeper.yml.disabled b/metricbeat/modules.d/zookeeper.yml.disabled new file mode 100644 index 000000000000..63543cefdd71 --- /dev/null +++ b/metricbeat/modules.d/zookeeper.yml.disabled @@ -0,0 +1,4 @@ +- module: zookeeper + metricsets: ["mntr"] + period: 10s + hosts: ["localhost:2181"] diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index d9ff62b19439..4ca002b67e22 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -93,7 +93,7 @@ metricbeat.modules: {% if reload -%} metricbeat.config.modules: - path: {{ reload_path }} + path: {{ reload_path|default("${path.config}/modules.d/*.yml") }} reload.period: 1s reload.enabled: true {% endif -%} diff --git a/metricbeat/tests/system/test_cmd.py b/metricbeat/tests/system/test_cmd.py new file mode 100644 index 000000000000..a3657b42d702 --- /dev/null +++ b/metricbeat/tests/system/test_cmd.py @@ -0,0 +1,111 @@ +import os +import metricbeat + + +class TestCommands(metricbeat.BaseTest): + """ + Test metricbeat subcommands + """ + + def setUp(self): + super(TestCommands, self).setUp() + + # Enable modules reload with default paths + self.render_config_template(reload=True) + os.mkdir(self.working_dir + "/modules.d") + + def test_modules_list(self): + """ + Test modules list command + """ + self.touch(self.working_dir + "/modules.d/enabled.yml") + self.touch(self.working_dir + "/modules.d/disabled.yml.disabled") + + exit_code = self.run_beat(extra_args=["modules", "list"]) + + assert exit_code == 0 + assert "Enabled:\nenabled" in self.get_log() + assert "Disabled:\ndisabled" in self.get_log() + + # Add one more disabled module + self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled") + exit_code = self.run_beat(extra_args=["modules", "list"]) + + assert exit_code == 0 + assert "Enabled:\nenabled" in self.get_log() + assert "Disabled:\ndisabled\ndisabled2" in self.get_log() + + def test_modules_enable(self): + """ + Test modules enable command + """ + self.touch(self.working_dir + "/modules.d/enabled.yml") + self.touch(self.working_dir + "/modules.d/disabled1.yml.disabled") + self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled") + self.touch(self.working_dir + "/modules.d/disabled3.yml.disabled") + + # Enable one module + exit_code = self.run_beat( + extra_args=["modules", "enable", "disabled1"]) + assert exit_code == 0 + + assert self.log_contains("Enabled disabled1") + assert os.path.exists(self.working_dir + "/modules.d/disabled1.yml") + assert not os.path.exists( + self.working_dir + "/modules.d/disabled1.yml.disabled") + assert os.path.exists( + self.working_dir + "/modules.d/disabled2.yml.disabled") + assert os.path.exists( + self.working_dir + "/modules.d/disabled3.yml.disabled") + + # Enable several modules at once: + exit_code = self.run_beat( + extra_args=["modules", "enable", "disabled2", "disabled3"]) + assert exit_code == 0 + + assert self.log_contains("Enabled disabled2") + assert self.log_contains("Enabled disabled3") + assert os.path.exists(self.working_dir + "/modules.d/disabled2.yml") + assert os.path.exists(self.working_dir + "/modules.d/disabled3.yml") + assert not os.path.exists( + self.working_dir + "/modules.d/disabled2.yml.disabled") + assert not os.path.exists( + self.working_dir + "/modules.d/disabled3.yml.disabled") + + def test_modules_disable(self): + """ + Test modules disable command + """ + self.touch(self.working_dir + "/modules.d/enabled1.yml") + self.touch(self.working_dir + "/modules.d/enabled2.yml") + self.touch(self.working_dir + "/modules.d/enabled3.yml") + self.touch(self.working_dir + "/modules.d/disabled2.yml.disabled") + + # Disable one module + exit_code = self.run_beat( + extra_args=["modules", "disable", "enabled1"]) + assert exit_code == 0 + + assert self.log_contains("Disabled enabled1") + assert os.path.exists( + self.working_dir + "/modules.d/enabled1.yml.disabled") + assert not os.path.exists(self.working_dir + "/modules.d/enabled1.yml") + assert os.path.exists(self.working_dir + "/modules.d/enabled2.yml") + assert os.path.exists(self.working_dir + "/modules.d/enabled3.yml") + + # Disable several modules at once: + exit_code = self.run_beat( + extra_args=["modules", "disable", "enabled2", "enabled3"]) + assert exit_code == 0 + + assert self.log_contains("Disabled enabled2") + assert self.log_contains("Disabled enabled3") + assert os.path.exists( + self.working_dir + "/modules.d/enabled2.yml.disabled") + assert os.path.exists( + self.working_dir + "/modules.d/enabled3.yml.disabled") + assert not os.path.exists(self.working_dir + "/modules.d/enabled2.yml") + assert not os.path.exists(self.working_dir + "/modules.d/enabled3.yml") + + def touch(self, path): + open(path, 'a').close()