Add secondary source of modules to Metricbeat to read light modules

CHANGELOG.next.asciidoc

@@ -252,6 +252,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Always report Pod UID in the `pod` metricset. {pull}12345[12345]
 - Add Vsphere Virtual Machine operating system to `os` field in Vsphere virtualmachine module. {pull}12391[12391]
 - Add validation for elasticsearch and kibana modules' metricsets when xpack.enabled is set to true. {pull}12386[12386]
+- Add support for metricbeat modules based on existing modules (a.k.a. light modules) {issue}12270[12270] {pull}12465[12465]
 - Add a system/entropy metricset {pull}12450[12450]
 - Allow redis URL format in redis hosts config. {pull}12408[12408]
 

metricbeat/beater/metricbeat.go

@@ -99,9 +99,6 @@ func DefaultCreator() beat.Creator {
 
 // newMetricbeat creates and returns a new Metricbeat instance.
 func newMetricbeat(b *beat.Beat, c *common.Config, options ...Option) (*Metricbeat, error) {
-	// List all registered modules and metricsets.
-	logp.Debug("modules", "%s", mb.Registry.String())
-
 	config := defaultConfig
 	if err := c.Unpack(&config); err != nil {
 		return nil, errors.Wrap(err, "error reading configuration file")
@@ -120,6 +117,9 @@ func newMetricbeat(b *beat.Beat, c *common.Config, options ...Option) (*Metricbe
 		applyOption(metricbeat)
 	}
 
+	// List all registered modules and metricsets.
+	logp.Debug("modules", "Available modules and metricsets: %s", mb.Registry.String())
+
 	if b.InSetupCmd {
 		// Return without instantiating the metricsets.
 		return metricbeat, nil

metricbeat/cmd/modules.go

@@ -27,7 +27,8 @@ import (
 	"github.com/elastic/beats/libbeat/cmd"
 )
 
-func buildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
+// BuildModulesManager adds support for modules management to a beat
+func BuildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
 	config := beat.BeatConfig
 
 	glob, err := config.String("config.modules.path", -1)

metricbeat/cmd/root.go

@@ -26,6 +26,7 @@ import (
 	"github.com/elastic/beats/libbeat/cmd/instance"
 	"github.com/elastic/beats/metricbeat/beater"
 	"github.com/elastic/beats/metricbeat/cmd/test"
+	"github.com/elastic/beats/metricbeat/mb/module"
 
 	// import modules
 	_ "github.com/elastic/beats/metricbeat/include"
@@ -38,10 +39,22 @@ var Name = "metricbeat"
 // RootCmd to handle beats cli
 var RootCmd *cmd.BeatsRootCmd
 
+var (
+	// Use a customized instance of Metricbeat where startup delay has
+	// been disabled to workaround the fact that Modules() will return
+	// the static modules (not the dynamic ones) with a start delay.
+	testModulesCreator = beater.Creator(
+		beater.WithModuleOptions(
+			module.WithMetricSetInfo(),
+			module.WithMaxStartDelay(0),
+		),
+	)
+)
+
 func init() {
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("system.hostfs"))
 	RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), instance.Settings{RunFlags: runFlags, Name: Name})
-	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
-	RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, ""))
+	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", BuildModulesManager))
+	RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", testModulesCreator))
 }

metricbeat/cmd/test/modules.go

@@ -23,13 +23,13 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/cmd/instance"
 	"github.com/elastic/beats/libbeat/testing"
 	"github.com/elastic/beats/metricbeat/beater"
-	"github.com/elastic/beats/metricbeat/mb/module"
 )
 
-func GenTestModulesCmd(name, beatVersion string) *cobra.Command {
+func GenTestModulesCmd(name, beatVersion string, create beat.Creator) *cobra.Command {
 	return &cobra.Command{
 		Use:   "modules [module] [metricset]",
 		Short: "Test modules settings",
@@ -49,15 +49,6 @@ func GenTestModulesCmd(name, beatVersion string) *cobra.Command {
 				os.Exit(1)
 			}
 
-			// Use a customized instance of Metricbeat where startup delay has
-			// been disabled to workaround the fact that Modules() will return
-			// the static modules (not the dynamic ones) with a start delay.
-			create := beater.Creator(
-				beater.WithModuleOptions(
-					module.WithMetricSetInfo(),
-					module.WithMaxStartDelay(0),
-				),
-			)
 			mb, err := create(&b.Beat, b.Beat.BeatConfig)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error initializing metricbeat: %s\n", err)

metricbeat/mb/lightmetricset.go

@@ -0,0 +1,117 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mb
+
+import (
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/common"
+)
+
+// LightMetricSet contains the definition of a non-registered metric set
+type LightMetricSet struct {
+	Name    string
+	Module  string
+	Default bool `config:"default"`
+	Input   struct {
+		Module    string      `config:"module" validate:"required"`
+		MetricSet string      `config:"metricset" validate:"required"`
+		Defaults  interface{} `config:"defaults"`
+	} `config:"input" validate:"required"`
+}
+
+// Registration obtains a metric set registration for this light metric set, this registration
+// contains a metric set factory that reprocess metric set creation taking into account the
+// light metric set defaults
+func (m *LightMetricSet) Registration(r *Register) (MetricSetRegistration, error) {
+	registration, err := r.metricSetRegistration(m.Input.Module, m.Input.MetricSet)
+	if err != nil {
+		return registration, errors.Wrapf(err,
+			"failed to start light metricset '%s/%s' using '%s/%s' metricset as input",
+			m.Module, m.Name,
+			m.Input.Module, m.Input.MetricSet)
+	}
+
+	originalFactory := registration.Factory
+	registration.IsDefault = m.Default
+
+	// Light modules factory has to override defaults and reproduce builder
+	// functionality with the resulting configuration, it does:
+	// - Override defaults
+	// - Call module factory if registered (it wouldn't have been called
+	//   if light module is really a registered mixed module)
+	// - Call host parser if defined (it would have already been called
+	//   without the light module defaults)
+	// - Finally, call the original factory for the registered metricset
+	registration.Factory = func(base BaseMetricSet) (MetricSet, error) {
+		// Override default config on base module and metricset
+		base.name = m.Name
+		baseModule, err := m.baseModule(base.module)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create base module for light module '%s', using base module '%s'", m.Module, base.module.Name())
+		}
+		base.module = baseModule
+
+		// Run module factory if registered, it will be called once per
+		// metricset, but it should be idempotent
+		moduleFactory := r.moduleFactory(m.Input.Module)
+		if moduleFactory != nil {
+			module, err := moduleFactory(*baseModule)
+			if err != nil {
+				return nil, errors.Wrapf(err, "module factory for module '%s' failed while creating light metricset '%s/%s'", m.Input.Module, m.Module, m.Name)
+			}
+			base.module = module
+		}
+
+		// At this point host parser was already run, we need to run this again
+		// with the overriden defaults
+		if registration.HostParser != nil {
+			base.hostData, err = registration.HostParser(base.module, base.host)
+			if err != nil {
+				return nil, errors.Wrapf(err, "host parser failed on light metricset factory for '%s/%s'", m.Module, m.Name)
+			}
+			base.host = base.hostData.Host
+		}
+
+		return originalFactory(base)
+	}
+
+	return registration, nil
+}
+
+// baseModule does the configuration overrides in the base module configuration
+// taking into account the light metric set default configurations
+func (m *LightMetricSet) baseModule(from Module) (*BaseModule, error) {
+	baseModule := BaseModule{
+		name: m.Module,
+	}
+	var err error
+	// Set defaults
+	if baseModule.rawConfig, err = common.NewConfigFrom(m.Input.Defaults); err != nil {
+		return nil, errors.Wrap(err, "invalid input defaults")
+	}
+	// Copy values from user configuration
+	if err = from.UnpackConfig(baseModule.rawConfig); err != nil {
+		return nil, errors.Wrap(err, "failed to copy values from user configuration")
+	}
+	// Update module configuration
+	if err = baseModule.UnpackConfig(&baseModule.config); err != nil {
+		return nil, errors.Wrap(err, "failed to set module configuration")
+	}
+	return &baseModule, nil
+}

metricbeat/mb/lightmetricset_test.go

@@ -0,0 +1,139 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mb
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/beats/libbeat/common"
+)
+
+func TestLightMetricSetRegistration(t *testing.T) {
+	cases := map[string]struct {
+		module    string
+		metricSet string
+		isDefault bool
+		fail      bool
+	}{
+		"metricset is registered": {
+			module:    "foo",
+			metricSet: "bar",
+			fail:      false,
+		},
+		"metricset is registered and is default": {
+			module:    "foo",
+			metricSet: "bar",
+			isDefault: true,
+			fail:      false,
+		},
+		"module is not registered": {
+			module:    "notexists",
+			metricSet: "notexists",
+			fail:      true,
+		},
+		"metricset is not registered": {
+			module:    "foo",
+			metricSet: "notexists",
+			fail:      true,
+		},
+	}
+
+	fakeMetricSetFactory := func(b BaseMetricSet) (MetricSet, error) { return &b, nil }
+
+	moduleName := "foo"
+	metricSetName := "bar"
+	lightMetricSetName := "metricset"
+	lightModuleName := "module"
+
+	r := NewRegister()
+	r.MustAddMetricSet(moduleName, metricSetName, fakeMetricSetFactory)
+
+	for title, c := range cases {
+		t.Run(title, func(t *testing.T) {
+			ms := LightMetricSet{
+				Name:    lightMetricSetName,
+				Module:  lightModuleName,
+				Default: c.isDefault,
+			}
+			ms.Input.Module = c.module
+			ms.Input.MetricSet = c.metricSet
+			ms.Input.Defaults = common.MapStr{
+				"query": common.MapStr{
+					"extra": "something",
+				},
+			}
+
+			registration, err := ms.Registration(r)
+			if c.fail {
+				assert.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			// Check that registration has the light metricset settings
+			assert.Equal(t, c.metricSet, registration.Name)
+			assert.Equal(t, c.isDefault, registration.IsDefault)
+
+			// Check that calling the factory with a registered base module:
+			// - Does not modify original base module
+			// - Does the proper overrides in the resulting metricset
+			bm := baseModule(t, r, moduleName, metricSetName)
+			moduleConfigBefore := bm.Module().Config().String()
+			metricSet, err := registration.Factory(bm)
+
+			assert.Equal(t, moduleConfigBefore, bm.Module().Config().String(),
+				"original base module config should not change")
+			require.NoError(t, err)
+			require.NotNil(t, metricSet)
+
+			assert.Equal(t, lightModuleName, metricSet.Module().Name())
+			assert.Equal(t, lightMetricSetName, metricSet.Name())
+
+			expectedQuery := QueryParams{
+				"default": "foo",
+				"extra":   "something",
+			}
+			query := metricSet.Module().Config().Query
+			assert.Equal(t, expectedQuery, query)
+		})
+	}
+}
+
+func baseModule(t *testing.T, r *Register, module, metricSet string) BaseMetricSet {
+	origRegistration, err := r.metricSetRegistration(module, metricSet)
+	require.NoError(t, err)
+
+	c := DefaultModuleConfig()
+	c.Module = module
+	c.MetricSets = []string{metricSet}
+	c.Query = QueryParams{"default": "foo"}
+	raw, err := common.NewConfigFrom(c)
+	require.NoError(t, err)
+	baseModule, err := newBaseModuleFromConfig(raw)
+	require.NoError(t, err)
+
+	bm := BaseMetricSet{
+		name:         "bar",
+		module:       &baseModule,
+		registration: origRegistration,
+	}
+	return bm
+}