diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 92803585fb4..1f07e2829c8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -233,6 +233,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Always report Pod UID in the `pod` metricset. {pull}12345[12345] - Add Vsphere Virtual Machine operating system to `os` field in Vsphere virtualmachine module. {pull}12391[12391] - Add validation for elasticsearch and kibana modules' metricsets when xpack.enabled is set to true. {pull}12386[12386] +- Add support for metricbeat modules based on existing modules (a.k.a. light modules) {issue}12270[12270] {pull}12465[12465] - Add a system/entropy metricset {pull}12450[12450] - Allow redis URL format in redis hosts config. {pull}12408[12408] - Add tags into ec2 metricset. {issue}[12263]12263 {pull}12372[12372] diff --git a/metricbeat/beater/metricbeat.go b/metricbeat/beater/metricbeat.go index 7703720521c..eb8e61f70ab 100644 --- a/metricbeat/beater/metricbeat.go +++ b/metricbeat/beater/metricbeat.go @@ -99,9 +99,6 @@ func DefaultCreator() beat.Creator { // newMetricbeat creates and returns a new Metricbeat instance. func newMetricbeat(b *beat.Beat, c *common.Config, options ...Option) (*Metricbeat, error) { - // List all registered modules and metricsets. - logp.Debug("modules", "%s", mb.Registry.String()) - config := defaultConfig if err := c.Unpack(&config); err != nil { return nil, errors.Wrap(err, "error reading configuration file") @@ -120,6 +117,9 @@ func newMetricbeat(b *beat.Beat, c *common.Config, options ...Option) (*Metricbe applyOption(metricbeat) } + // List all registered modules and metricsets. + logp.Debug("modules", "Available modules and metricsets: %s", mb.Registry.String()) + if b.InSetupCmd { // Return without instantiating the metricsets. return metricbeat, nil diff --git a/metricbeat/cmd/modules.go b/metricbeat/cmd/modules.go index 08362356ee6..608eb47fc7b 100644 --- a/metricbeat/cmd/modules.go +++ b/metricbeat/cmd/modules.go @@ -27,7 +27,8 @@ import ( "github.com/elastic/beats/libbeat/cmd" ) -func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) { +// BuildModulesManager adds support for modules management to a beat +func BuildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) { config := beat.BeatConfig glob, err := config.String("config.modules.path", -1) diff --git a/metricbeat/cmd/root.go b/metricbeat/cmd/root.go index da441482bad..9c0d24cc932 100644 --- a/metricbeat/cmd/root.go +++ b/metricbeat/cmd/root.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/metricbeat/beater" "github.com/elastic/beats/metricbeat/cmd/test" + "github.com/elastic/beats/metricbeat/mb/module" // import modules _ "github.com/elastic/beats/metricbeat/include" @@ -38,10 +39,22 @@ var Name = "metricbeat" // RootCmd to handle beats cli var RootCmd *cmd.BeatsRootCmd +var ( + // Use a customized instance of Metricbeat where startup delay has + // been disabled to workaround the fact that Modules() will return + // the static modules (not the dynamic ones) with a start delay. + testModulesCreator = beater.Creator( + beater.WithModuleOptions( + module.WithMetricSetInfo(), + module.WithMaxStartDelay(0), + ), + ) +) + func init() { var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), instance.Settings{RunFlags: runFlags, Name: Name}) - RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager)) - RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "")) + RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", BuildModulesManager)) + RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", testModulesCreator)) } diff --git a/metricbeat/cmd/test/modules.go b/metricbeat/cmd/test/modules.go index ef28beae23b..f2cafc5b43c 100644 --- a/metricbeat/cmd/test/modules.go +++ b/metricbeat/cmd/test/modules.go @@ -23,13 +23,13 @@ import ( "github.com/spf13/cobra" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/libbeat/testing" "github.com/elastic/beats/metricbeat/beater" - "github.com/elastic/beats/metricbeat/mb/module" ) -func GenTestModulesCmd(name, beatVersion string) *cobra.Command { +func GenTestModulesCmd(name, beatVersion string, create beat.Creator) *cobra.Command { return &cobra.Command{ Use: "modules [module] [metricset]", Short: "Test modules settings", @@ -49,15 +49,6 @@ func GenTestModulesCmd(name, beatVersion string) *cobra.Command { os.Exit(1) } - // Use a customized instance of Metricbeat where startup delay has - // been disabled to workaround the fact that Modules() will return - // the static modules (not the dynamic ones) with a start delay. - create := beater.Creator( - beater.WithModuleOptions( - module.WithMetricSetInfo(), - module.WithMaxStartDelay(0), - ), - ) mb, err := create(&b.Beat, b.Beat.BeatConfig) if err != nil { fmt.Fprintf(os.Stderr, "Error initializing metricbeat: %s\n", err) diff --git a/metricbeat/mb/lightmetricset.go b/metricbeat/mb/lightmetricset.go new file mode 100644 index 00000000000..eb0ed1d9d40 --- /dev/null +++ b/metricbeat/mb/lightmetricset.go @@ -0,0 +1,117 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mb + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" +) + +// LightMetricSet contains the definition of a non-registered metric set +type LightMetricSet struct { + Name string + Module string + Default bool `config:"default"` + Input struct { + Module string `config:"module" validate:"required"` + MetricSet string `config:"metricset" validate:"required"` + Defaults interface{} `config:"defaults"` + } `config:"input" validate:"required"` +} + +// Registration obtains a metric set registration for this light metric set, this registration +// contains a metric set factory that reprocess metric set creation taking into account the +// light metric set defaults +func (m *LightMetricSet) Registration(r *Register) (MetricSetRegistration, error) { + registration, err := r.metricSetRegistration(m.Input.Module, m.Input.MetricSet) + if err != nil { + return registration, errors.Wrapf(err, + "failed to start light metricset '%s/%s' using '%s/%s' metricset as input", + m.Module, m.Name, + m.Input.Module, m.Input.MetricSet) + } + + originalFactory := registration.Factory + registration.IsDefault = m.Default + + // Light modules factory has to override defaults and reproduce builder + // functionality with the resulting configuration, it does: + // - Override defaults + // - Call module factory if registered (it wouldn't have been called + // if light module is really a registered mixed module) + // - Call host parser if defined (it would have already been called + // without the light module defaults) + // - Finally, call the original factory for the registered metricset + registration.Factory = func(base BaseMetricSet) (MetricSet, error) { + // Override default config on base module and metricset + base.name = m.Name + baseModule, err := m.baseModule(base.module) + if err != nil { + return nil, errors.Wrapf(err, "failed to create base module for light module '%s', using base module '%s'", m.Module, base.module.Name()) + } + base.module = baseModule + + // Run module factory if registered, it will be called once per + // metricset, but it should be idempotent + moduleFactory := r.moduleFactory(m.Input.Module) + if moduleFactory != nil { + module, err := moduleFactory(*baseModule) + if err != nil { + return nil, errors.Wrapf(err, "module factory for module '%s' failed while creating light metricset '%s/%s'", m.Input.Module, m.Module, m.Name) + } + base.module = module + } + + // At this point host parser was already run, we need to run this again + // with the overriden defaults + if registration.HostParser != nil { + base.hostData, err = registration.HostParser(base.module, base.host) + if err != nil { + return nil, errors.Wrapf(err, "host parser failed on light metricset factory for '%s/%s'", m.Module, m.Name) + } + base.host = base.hostData.Host + } + + return originalFactory(base) + } + + return registration, nil +} + +// baseModule does the configuration overrides in the base module configuration +// taking into account the light metric set default configurations +func (m *LightMetricSet) baseModule(from Module) (*BaseModule, error) { + baseModule := BaseModule{ + name: m.Module, + } + var err error + // Set defaults + if baseModule.rawConfig, err = common.NewConfigFrom(m.Input.Defaults); err != nil { + return nil, errors.Wrap(err, "invalid input defaults") + } + // Copy values from user configuration + if err = from.UnpackConfig(baseModule.rawConfig); err != nil { + return nil, errors.Wrap(err, "failed to copy values from user configuration") + } + // Update module configuration + if err = baseModule.UnpackConfig(&baseModule.config); err != nil { + return nil, errors.Wrap(err, "failed to set module configuration") + } + return &baseModule, nil +} diff --git a/metricbeat/mb/lightmetricset_test.go b/metricbeat/mb/lightmetricset_test.go new file mode 100644 index 00000000000..ecb3f374d48 --- /dev/null +++ b/metricbeat/mb/lightmetricset_test.go @@ -0,0 +1,139 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mb + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/common" +) + +func TestLightMetricSetRegistration(t *testing.T) { + cases := map[string]struct { + module string + metricSet string + isDefault bool + fail bool + }{ + "metricset is registered": { + module: "foo", + metricSet: "bar", + fail: false, + }, + "metricset is registered and is default": { + module: "foo", + metricSet: "bar", + isDefault: true, + fail: false, + }, + "module is not registered": { + module: "notexists", + metricSet: "notexists", + fail: true, + }, + "metricset is not registered": { + module: "foo", + metricSet: "notexists", + fail: true, + }, + } + + fakeMetricSetFactory := func(b BaseMetricSet) (MetricSet, error) { return &b, nil } + + moduleName := "foo" + metricSetName := "bar" + lightMetricSetName := "metricset" + lightModuleName := "module" + + r := NewRegister() + r.MustAddMetricSet(moduleName, metricSetName, fakeMetricSetFactory) + + for title, c := range cases { + t.Run(title, func(t *testing.T) { + ms := LightMetricSet{ + Name: lightMetricSetName, + Module: lightModuleName, + Default: c.isDefault, + } + ms.Input.Module = c.module + ms.Input.MetricSet = c.metricSet + ms.Input.Defaults = common.MapStr{ + "query": common.MapStr{ + "extra": "something", + }, + } + + registration, err := ms.Registration(r) + if c.fail { + assert.Error(t, err) + return + } + require.NoError(t, err) + + // Check that registration has the light metricset settings + assert.Equal(t, c.metricSet, registration.Name) + assert.Equal(t, c.isDefault, registration.IsDefault) + + // Check that calling the factory with a registered base module: + // - Does not modify original base module + // - Does the proper overrides in the resulting metricset + bm := baseModule(t, r, moduleName, metricSetName) + moduleConfigBefore := bm.Module().Config().String() + metricSet, err := registration.Factory(bm) + + assert.Equal(t, moduleConfigBefore, bm.Module().Config().String(), + "original base module config should not change") + require.NoError(t, err) + require.NotNil(t, metricSet) + + assert.Equal(t, lightModuleName, metricSet.Module().Name()) + assert.Equal(t, lightMetricSetName, metricSet.Name()) + + expectedQuery := QueryParams{ + "default": "foo", + "extra": "something", + } + query := metricSet.Module().Config().Query + assert.Equal(t, expectedQuery, query) + }) + } +} + +func baseModule(t *testing.T, r *Register, module, metricSet string) BaseMetricSet { + origRegistration, err := r.metricSetRegistration(module, metricSet) + require.NoError(t, err) + + c := DefaultModuleConfig() + c.Module = module + c.MetricSets = []string{metricSet} + c.Query = QueryParams{"default": "foo"} + raw, err := common.NewConfigFrom(c) + require.NoError(t, err) + baseModule, err := newBaseModuleFromConfig(raw) + require.NoError(t, err) + + bm := BaseMetricSet{ + name: "bar", + module: &baseModule, + registration: origRegistration, + } + return bm +} diff --git a/metricbeat/mb/registry.go b/metricbeat/mb/registry.go index 51f98cb1c0f..591ea3692a5 100644 --- a/metricbeat/mb/registry.go +++ b/metricbeat/mb/registry.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/logp" ) @@ -105,6 +107,19 @@ type Register struct { modules map[string]ModuleFactory // A map of module name to nested map of MetricSet name to MetricSetRegistration. metricSets map[string]map[string]MetricSetRegistration + // Additional source of non-registered modules + secondarySource ModulesSource +} + +// ModulesSource contains a source of non-registered modules +type ModulesSource interface { + Modules() ([]string, error) + HasModule(module string) bool + MetricSets(module string) ([]string, error) + DefaultMetricSets(module string) ([]string, error) + HasMetricSet(module, name string) bool + MetricSetRegistration(r *Register, module, name string) (MetricSetRegistration, error) + String() string } // NewRegister creates and returns a new Register. @@ -222,16 +237,23 @@ func (r *Register) metricSetRegistration(module, name string) (MetricSetRegistra name = strings.ToLower(name) metricSets, exists := r.metricSets[module] - if !exists { - return MetricSetRegistration{}, fmt.Errorf("metricset '%s/%s' is not registered, module not found", module, name) + if exists { + registration, exists := metricSets[name] + if exists { + return registration, nil + } } - registration, exists := metricSets[name] - if !exists { - return MetricSetRegistration{}, fmt.Errorf("metricset '%s/%s' is not registered, metricset not found", module, name) + // Fallback to secondary source if module is not registered + if source := r.secondarySource; source != nil && source.HasMetricSet(module, name) { + registration, err := source.MetricSetRegistration(r, module, name) + if err != nil { + return MetricSetRegistration{}, errors.Wrapf(err, "failed to obtain registration for non-registered metricset '%s/%s'", module, name) + } + return registration, nil } - return registration, nil + return MetricSetRegistration{}, fmt.Errorf("metricset '%s/%s' not found", module, name) } // DefaultMetricSets returns the names of the default MetricSets for a module. @@ -243,18 +265,30 @@ func (r *Register) DefaultMetricSets(module string) ([]string, error) { module = strings.ToLower(module) + var defaults []string metricSets, exists := r.metricSets[module] - if !exists { - return nil, fmt.Errorf("module '%s' not found", module) + if exists { + for _, reg := range metricSets { + if reg.IsDefault { + defaults = append(defaults, reg.Name) + } + } } - var defaults []string - for _, reg := range metricSets { - if reg.IsDefault { - defaults = append(defaults, reg.Name) + // List also default metrics from secondary sources + if source := r.secondarySource; source != nil && source.HasModule(module) { + exists = true + sourceDefaults, err := source.DefaultMetricSets(module) + if err != nil { + logp.Error(errors.Wrapf(err, "failed to get default metric sets for module '%s' from secondary source", module)) + } else if len(sourceDefaults) > 0 { + defaults = append(defaults, sourceDefaults...) } } + if !exists { + return nil, fmt.Errorf("module '%s' not found", module) + } if len(defaults) == 0 { return nil, fmt.Errorf("no default metricset exists for module '%s'", module) } @@ -279,9 +313,10 @@ func (r *Register) MetricSets(module string) []string { r.lock.RLock() defer r.lock.RUnlock() - var metricsets []string + module = strings.ToLower(module) - sets, ok := r.metricSets[strings.ToLower(module)] + var metricsets []string + sets, ok := r.metricSets[module] if ok { metricsets = make([]string, 0, len(sets)) for name := range sets { @@ -289,9 +324,26 @@ func (r *Register) MetricSets(module string) []string { } } + // List also metric sets from secondary sources + if source := r.secondarySource; source != nil && source.HasModule(module) { + sourceMetricSets, err := source.MetricSets(module) + if err != nil { + logp.Error(errors.Wrap(err, "failed to get metricsets from secondary source")) + } + metricsets = append(metricsets, sourceMetricSets...) + } + return metricsets } +// SetSecondarySource sets an additional source of modules +func (r *Register) SetSecondarySource(source ModulesSource) { + r.lock.Lock() + defer r.lock.Unlock() + + r.secondarySource = source +} + // String return a string representation of the registered ModuleFactory's and // MetricSetFactory's. func (r *Register) String() string { @@ -302,7 +354,6 @@ func (r *Register) String() string { for module := range r.modules { modules = append(modules, module) } - sort.Strings(modules) var metricSets []string for module, m := range r.metricSets { @@ -310,8 +361,14 @@ func (r *Register) String() string { metricSets = append(metricSets, fmt.Sprintf("%s/%s", module, name)) } } - sort.Strings(metricSets) - return fmt.Sprintf("Register [ModuleFactory:[%s], MetricSetFactory:[%s]]", - strings.Join(modules, ", "), strings.Join(metricSets, ", ")) + var secondarySource string + if source := r.secondarySource; source != nil { + secondarySource = ", " + source.String() + } + + sort.Strings(modules) + sort.Strings(metricSets) + return fmt.Sprintf("Register [ModuleFactory:[%s], MetricSetFactory:[%s]%s]", + strings.Join(modules, ", "), strings.Join(metricSets, ", "), secondarySource) } diff --git a/metricbeat/tests/system/test_reload.py b/metricbeat/tests/system/test_reload.py index ad88ba4654a..02d29d175be 100644 --- a/metricbeat/tests/system/test_reload.py +++ b/metricbeat/tests/system/test_reload.py @@ -114,7 +114,7 @@ def test_wrong_module_no_reload(self): # Wait until offset for new line is updated self.wait_until( - lambda: self.log_contains("metricset not found"), + lambda: self.log_contains("metricset 'system/wrong_metricset' not found"), max_timeout=10) assert exit_code == 1 diff --git a/x-pack/metricbeat/beater/metricbeat.go b/x-pack/metricbeat/beater/metricbeat.go new file mode 100644 index 00000000000..09069aa2f26 --- /dev/null +++ b/x-pack/metricbeat/beater/metricbeat.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package beater + +import ( + "github.com/elastic/beats/libbeat/paths" + "github.com/elastic/beats/metricbeat/beater" + "github.com/elastic/beats/metricbeat/mb" + xpackmb "github.com/elastic/beats/x-pack/metricbeat/mb" +) + +// WithLightModules enables light modules support +func WithLightModules() beater.Option { + return func(*beater.Metricbeat) { + path := paths.Resolve(paths.Home, "module") + mb.Registry.SetSecondarySource(xpackmb.NewLightModulesSource(path)) + } +} diff --git a/x-pack/metricbeat/cmd/root.go b/x-pack/metricbeat/cmd/root.go index 3a931232fa7..684ce920f24 100644 --- a/x-pack/metricbeat/cmd/root.go +++ b/x-pack/metricbeat/cmd/root.go @@ -5,16 +5,59 @@ package cmd import ( - "github.com/elastic/beats/metricbeat/cmd" + "flag" + + "github.com/spf13/pflag" + + cmd "github.com/elastic/beats/libbeat/cmd" + "github.com/elastic/beats/libbeat/cmd/instance" + "github.com/elastic/beats/metricbeat/beater" + mbcmd "github.com/elastic/beats/metricbeat/cmd" + "github.com/elastic/beats/metricbeat/cmd/test" + "github.com/elastic/beats/metricbeat/mb/module" xpackcmd "github.com/elastic/beats/x-pack/libbeat/cmd" + xpackbeater "github.com/elastic/beats/x-pack/metricbeat/beater" // Register the includes. _ "github.com/elastic/beats/x-pack/metricbeat/include" + + // Import OSS modules. + _ "github.com/elastic/beats/metricbeat/include" + _ "github.com/elastic/beats/metricbeat/include/fields" ) +// Name of this beat +var Name = "metricbeat" + // RootCmd to handle beats cli -var RootCmd = cmd.RootCmd +var RootCmd *cmd.BeatsRootCmd + +var ( + rootCreator = beater.Creator( + xpackbeater.WithLightModules(), + beater.WithModuleOptions( + module.WithMetricSetInfo(), + module.WithServiceName(), + ), + ) + + // Use a customized instance of Metricbeat where startup delay has + // been disabled to workaround the fact that Modules() will return + // the static modules (not the dynamic ones) with a start delay. + testModulesCreator = beater.Creator( + xpackbeater.WithLightModules(), + beater.WithModuleOptions( + module.WithMetricSetInfo(), + module.WithMaxStartDelay(0), + ), + ) +) func init() { - xpackcmd.AddXPack(RootCmd, cmd.Name) + var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError) + runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs")) + RootCmd = cmd.GenRootCmdWithSettings(rootCreator, instance.Settings{RunFlags: runFlags, Name: Name}) + RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", mbcmd.BuildModulesManager)) + RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", testModulesCreator)) + xpackcmd.AddXPack(RootCmd, Name) } diff --git a/x-pack/metricbeat/main_test.go b/x-pack/metricbeat/main_test.go index 09351fa1fef..c022c48402c 100644 --- a/x-pack/metricbeat/main_test.go +++ b/x-pack/metricbeat/main_test.go @@ -8,7 +8,7 @@ import ( "flag" "testing" - "github.com/elastic/beats/metricbeat/cmd" + "github.com/elastic/beats/x-pack/metricbeat/cmd" ) var systemTest *bool diff --git a/x-pack/metricbeat/mb/lightmodules.go b/x-pack/metricbeat/mb/lightmodules.go new file mode 100644 index 00000000000..d01d887a72d --- /dev/null +++ b/x-pack/metricbeat/mb/lightmodules.go @@ -0,0 +1,239 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package mb + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" +) + +const ( + moduleYML = "module.yml" + manifestYML = "manifest.yml" +) + +// LightModulesSource loads module definitions from files in the provided paths +type LightModulesSource struct { + paths []string +} + +// NewLightModulesSource creates a new LightModulesSource +func NewLightModulesSource(paths ...string) *LightModulesSource { + return &LightModulesSource{ + paths: paths, + } +} + +// Modules lists the light modules available on the configured paths +func (s *LightModulesSource) Modules() ([]string, error) { + return s.moduleNames() +} + +// HasModule checks if there is a light module with the given name +func (s *LightModulesSource) HasModule(moduleName string) bool { + names, err := s.moduleNames() + if err != nil { + logp.Error(errors.Wrap(err, "failed to get list of light module names")) + return false + } + for _, name := range names { + if name == moduleName { + return true + } + } + return false +} + +// DefaultMetricSets list the default metricsets for a given module +func (s *LightModulesSource) DefaultMetricSets(moduleName string) ([]string, error) { + module, err := s.loadModule(moduleName) + if err != nil { + return nil, errors.Wrapf(err, "failed to get default metricsets for module '%s'", moduleName) + } + var metricsets []string + for _, ms := range module.MetricSets { + if ms.Default { + metricsets = append(metricsets, ms.Name) + } + } + return metricsets, nil +} + +// MetricSets list the available metricsets for a given module +func (s *LightModulesSource) MetricSets(moduleName string) ([]string, error) { + module, err := s.loadModule(moduleName) + if err != nil { + return nil, errors.Wrapf(err, "failed to get metricsets for module '%s'", moduleName) + } + metricsets := make([]string, 0, len(module.MetricSets)) + for _, ms := range module.MetricSets { + metricsets = append(metricsets, ms.Name) + } + return metricsets, nil +} + +// HasMetricSet checks if the given metricset exists +func (s *LightModulesSource) HasMetricSet(moduleName, metricSetName string) bool { + modulePath, found := s.findModulePath(moduleName) + if !found { + return false + } + + moduleConfig, err := s.loadModuleConfig(modulePath) + if err != nil { + logp.Error(errors.Wrapf(err, "failed to load module config for module '%s'", moduleName)) + return false + } + + for _, name := range moduleConfig.MetricSets { + if name == metricSetName { + return true + } + } + return false +} + +// MetricSetRegistration obtains a registration for a light metric set +func (s *LightModulesSource) MetricSetRegistration(register *mb.Register, moduleName, metricSetName string) (mb.MetricSetRegistration, error) { + lightModule, err := s.loadModule(moduleName) + if err != nil { + return mb.MetricSetRegistration{}, errors.Wrapf(err, "failed to load module '%s'", moduleName) + } + + ms, found := lightModule.MetricSets[metricSetName] + if !found { + return mb.MetricSetRegistration{}, fmt.Errorf("metricset '%s/%s' not found", moduleName, metricSetName) + } + + return ms.Registration(register) +} + +// String returns a string representation of this source, with a list of known metricsets +func (s *LightModulesSource) String() string { + var metricSets []string + modules, _ := s.Modules() + for _, module := range modules { + moduleMetricSets, _ := s.MetricSets(module) + for _, name := range moduleMetricSets { + metricSets = append(metricSets, fmt.Sprintf("%s/%s", module, name)) + } + } + + return fmt.Sprintf("LightModules:[%s]", strings.Join(metricSets, ", ")) +} + +type lightModuleConfig struct { + Name string `config:"name"` + MetricSets []string `config:"metricsets"` +} + +// LightModule contains the definition of a light module +type LightModule struct { + Name string + MetricSets map[string]mb.LightMetricSet +} + +func (s *LightModulesSource) loadModule(moduleName string) (*LightModule, error) { + modulePath, found := s.findModulePath(moduleName) + if !found { + return nil, fmt.Errorf("module '%s' not found", moduleName) + } + + moduleConfig, err := s.loadModuleConfig(modulePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to load light module '%s' definition", moduleName) + } + + metricSets, err := s.loadMetricSets(filepath.Dir(modulePath), moduleConfig.Name, moduleConfig.MetricSets) + if err != nil { + return nil, errors.Wrapf(err, "failed to load metric sets for light module '%s'", moduleName) + } + + return &LightModule{Name: moduleName, MetricSets: metricSets}, nil +} + +func (s *LightModulesSource) findModulePath(moduleName string) (string, bool) { + for _, dir := range s.paths { + candidate := filepath.Join(dir, moduleName, moduleYML) + if _, err := os.Stat(candidate); err == nil { + return candidate, true + } + } + return "", false +} + +func (s *LightModulesSource) loadModuleConfig(modulePath string) (*lightModuleConfig, error) { + config, err := common.LoadFile(modulePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to load module configuration from '%s'", modulePath) + } + + var moduleConfig lightModuleConfig + if err = config.Unpack(&moduleConfig); err != nil { + return nil, errors.Wrapf(err, "failed to parse light module definition from '%s'", modulePath) + } + return &moduleConfig, nil +} + +func (s *LightModulesSource) loadMetricSets(moduleDirPath, moduleName string, metricSetNames []string) (map[string]mb.LightMetricSet, error) { + metricSets := make(map[string]mb.LightMetricSet) + for _, metricSet := range metricSetNames { + manifestPath := filepath.Join(moduleDirPath, metricSet, manifestYML) + + metricSetConfig, err := s.loadMetricSetConfig(manifestPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to load light metricset '%s'", metricSet) + } + metricSetConfig.Name = metricSet + metricSetConfig.Module = moduleName + + metricSets[metricSet] = metricSetConfig + } + return metricSets, nil +} + +func (s *LightModulesSource) loadMetricSetConfig(manifestPath string) (ms mb.LightMetricSet, err error) { + config, err := common.LoadFile(manifestPath) + if err != nil { + return ms, errors.Wrapf(err, "failed to load metricset manifest from '%s'", manifestPath) + } + + if err := config.Unpack(&ms); err != nil { + return ms, errors.Wrapf(err, "failed to parse metricset manifest from '%s'", manifestPath) + } + return +} + +func (s *LightModulesSource) moduleNames() ([]string, error) { + modules := make(map[string]bool) + for _, dir := range s.paths { + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil, errors.Wrapf(err, "failed to list modules on path '%s'", dir) + } + for _, f := range files { + modulePath := filepath.Join(dir, f.Name(), moduleYML) + if _, err := os.Stat(modulePath); os.IsNotExist(err) { + continue + } + modules[f.Name()] = true + } + } + + names := make([]string, 0, len(modules)) + for name := range modules { + names = append(names, name) + } + return names, nil +} diff --git a/x-pack/metricbeat/mb/lightmodules_test.go b/x-pack/metricbeat/mb/lightmodules_test.go new file mode 100644 index 00000000000..e315274d7e4 --- /dev/null +++ b/x-pack/metricbeat/mb/lightmodules_test.go @@ -0,0 +1,303 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package mb + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" +) + +// TestLightModulesAsModuleSource checks that registry correctly lists +// metricsets when used with light modules +func TestLightModulesAsModuleSource(t *testing.T) { + logp.TestingSetup() + + type testMetricSet struct { + name string + module string + isDefault bool + hostParser mb.HostParser + } + + cases := map[string]struct { + registered []testMetricSet + expectedMetricSets map[string][]string + expectedDefaultMetricSets map[string][]string + }{ + "no registered modules": { + expectedMetricSets: map[string][]string{ + "service": []string{"metricset", "nondefault"}, + "broken": []string{}, + "empty": []string{}, + }, + expectedDefaultMetricSets: map[string][]string{ + "service": []string{"metricset"}, + "broken": []string{}, + "empty": []string{}, + }, + }, + "same module registered (mixed modules case)": { + registered: []testMetricSet{ + {name: "other", module: "service"}, + }, + expectedMetricSets: map[string][]string{ + "service": []string{"metricset", "nondefault", "other"}, + }, + expectedDefaultMetricSets: map[string][]string{ + "service": []string{"metricset"}, + }, + }, + "some metricsets registered": { + registered: []testMetricSet{ + {name: "other", module: "service"}, + {name: "metricset", module: "something", isDefault: true}, + {name: "metricset", module: "someotherthing"}, + }, + expectedMetricSets: map[string][]string{ + "service": []string{"metricset", "nondefault", "other"}, + "something": []string{"metricset"}, + "someotherthing": []string{"metricset"}, + }, + expectedDefaultMetricSets: map[string][]string{ + "service": []string{"metricset"}, + "something": []string{"metricset"}, + "someotherthing": []string{}, + }, + }, + } + + fakeMetricSetFactory := func(base mb.BaseMetricSet) (mb.MetricSet, error) { + return &base, nil + } + + newRegistry := func(metricSets []testMetricSet) *mb.Register { + r := mb.NewRegister() + for _, m := range metricSets { + opts := []mb.MetricSetOption{} + if m.isDefault { + opts = append(opts, mb.DefaultMetricSet()) + } + if m.hostParser != nil { + opts = append(opts, mb.WithHostParser(m.hostParser)) + } + r.MustAddMetricSet(m.module, m.name, fakeMetricSetFactory, opts...) + } + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + return r + } + + for title, c := range cases { + t.Run(title, func(t *testing.T) { + r := newRegistry(c.registered) + + // Check metricsets + for module, metricSets := range c.expectedMetricSets { + t.Run("metricsets for "+module, func(t *testing.T) { + assert.ElementsMatch(t, metricSets, r.MetricSets(module)) + }) + } + + // Check default metricsets + for module, expected := range c.expectedDefaultMetricSets { + t.Run("default metricsets for "+module, func(t *testing.T) { + found, err := r.DefaultMetricSets(module) + if len(expected) > 0 { + assert.NoError(t, err) + assert.ElementsMatch(t, expected, found) + } else { + assert.Error(t, err, "error expected when there are no default metricsets") + + } + }) + } + }) + } +} + +func TestLoadModule(t *testing.T) { + logp.TestingSetup() + + cases := []struct { + name string + exists bool + err bool + }{ + { + name: "service", + exists: true, + err: false, + }, + { + name: "broken", + exists: true, + err: true, + }, + { + name: "empty", + exists: false, + err: false, + }, + { + name: "notexists", + exists: false, + err: false, + }, + } + + for _, c := range cases { + r := NewLightModulesSource("testdata/lightmodules") + t.Run(c.name, func(t *testing.T) { + _, err := r.loadModule(c.name) + if c.err { + assert.Error(t, err) + } + assert.Equal(t, c.exists, r.HasModule(c.name)) + }) + } +} + +func TestNewModuleFromConfig(t *testing.T) { + logp.TestingSetup() + + cases := map[string]struct { + config common.MapStr + err bool + expectedOption string + expectedQuery mb.QueryParams + expectedPeriod time.Duration + }{ + "normal module": { + config: common.MapStr{"module": "foo", "metricsets": []string{"bar"}}, + expectedOption: "default", + expectedPeriod: mb.DefaultModuleConfig().Period, + }, + "light module": { + config: common.MapStr{"module": "service", "metricsets": []string{"metricset"}}, + expectedOption: "test", + }, + "light module default metricset": { + config: common.MapStr{"module": "service"}, + expectedOption: "test", + }, + "light module override option": { + config: common.MapStr{"module": "service", "option": "overriden"}, + expectedOption: "overriden", + }, + "light module with query": { + config: common.MapStr{"module": "service", "query": common.MapStr{"param": "foo"}}, + expectedOption: "test", + expectedQuery: mb.QueryParams{"param": "foo"}, + }, + "light module with custom period": { + config: common.MapStr{"module": "service", "period": "42s"}, + expectedOption: "test", + expectedPeriod: 42 * time.Second, + }, + "light module is broken": { + config: common.MapStr{"module": "broken"}, + err: true, + }, + "light metric set doesn't exist": { + config: common.MapStr{"module": "service", "metricsets": []string{"notexists"}}, + err: true, + }, + "disabled light module": { + config: common.MapStr{"module": "service", "enabled": false}, + err: true, + }, + } + + r := mb.NewRegister() + r.MustAddMetricSet("foo", "bar", newMetricSetWithOption) + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + + for title, c := range cases { + t.Run(title, func(t *testing.T) { + config, err := common.NewConfigFrom(c.config) + require.NoError(t, err) + + module, metricSets, err := mb.NewModule(config, r) + if c.err { + assert.Error(t, err) + return + } + require.NoError(t, err) + + assert.Equal(t, c.config["module"].(string), module.Config().Module) + if metricSetNames, ok := c.config["metricsets"].([]string); ok { + assert.ElementsMatch(t, metricSetNames, module.Config().MetricSets) + } + + assert.NotEmpty(t, metricSets) + assert.NoError(t, err) + for _, ms := range metricSets { + t.Run(ms.Name(), func(t *testing.T) { + ms, ok := ms.(*metricSetWithOption) + require.True(t, ok) + assert.Equal(t, c.expectedOption, ms.Option) + assert.Equal(t, c.expectedQuery, ms.Module().Config().Query) + if c.expectedPeriod > 0 { + assert.Equal(t, c.expectedPeriod, ms.Module().Config().Period) + } + }) + } + }) + } +} + +func TestNewModulesCallModuleFactory(t *testing.T) { + logp.TestingSetup() + + r := mb.NewRegister() + r.MustAddMetricSet("foo", "bar", newMetricSetWithOption) + r.SetSecondarySource(NewLightModulesSource("testdata/lightmodules")) + + called := false + r.AddModule("foo", func(base mb.BaseModule) (mb.Module, error) { + called = true + return mb.DefaultModuleFactory(base) + }) + + config, err := common.NewConfigFrom(common.MapStr{"module": "service"}) + require.NoError(t, err) + + _, _, err = mb.NewModule(config, r) + assert.NoError(t, err) + + assert.True(t, called, "module factory must be called if registered") +} + +type metricSetWithOption struct { + mb.BaseMetricSet + Option string +} + +func newMetricSetWithOption(base mb.BaseMetricSet) (mb.MetricSet, error) { + config := struct { + Option string `config:"option"` + }{ + Option: "default", + } + err := base.Module().UnpackConfig(&config) + if err != nil { + return nil, err + } + + return &metricSetWithOption{ + BaseMetricSet: base, + Option: config.Option, + }, nil +} + +func (*metricSetWithOption) Fetch(mb.ReporterV2) error { return nil } diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/broken/module.yml b/x-pack/metricbeat/mb/testdata/lightmodules/broken/module.yml new file mode 100644 index 00000000000..a4b4e73c368 --- /dev/null +++ b/x-pack/metricbeat/mb/testdata/lightmodules/broken/module.yml @@ -0,0 +1,3 @@ +name: broken +metricsets: +- notexists diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/empty/.placeholder b/x-pack/metricbeat/mb/testdata/lightmodules/empty/.placeholder new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml b/x-pack/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml new file mode 100644 index 00000000000..5291cac44e6 --- /dev/null +++ b/x-pack/metricbeat/mb/testdata/lightmodules/service/metricset/manifest.yml @@ -0,0 +1,6 @@ +default: true +input: + module: foo + metricset: bar + defaults: + option: test diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/service/module.yml b/x-pack/metricbeat/mb/testdata/lightmodules/service/module.yml new file mode 100644 index 00000000000..7d037081eeb --- /dev/null +++ b/x-pack/metricbeat/mb/testdata/lightmodules/service/module.yml @@ -0,0 +1,4 @@ +name: service +metricsets: +- metricset +- nondefault diff --git a/x-pack/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml b/x-pack/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml new file mode 100644 index 00000000000..f4dc493d29c --- /dev/null +++ b/x-pack/metricbeat/mb/testdata/lightmodules/service/nondefault/manifest.yml @@ -0,0 +1,4 @@ +default: false +input: + module: foo + metricset: baz