[Generator] Fix metricbeat-style custom beats (elastic#13293)

* first pass at fixing cmd/init system of user-generated metricbeat

* remove old hack

* add lic header

* Trying to fix vendor

* get includes working, rename files

* use git archive, add template

* remove newline

generator/metricbeat/{beat}/Makefile

@@ -22,12 +22,12 @@ setup: copy-vendor git-init
 # Copy beats into vendor directory
 .PHONY: copy-vendor
 copy-vendor:
-	mkdir -p vendor/github.com/elastic
-	cp -R ${GOPATH}/src/github.com/elastic/beats vendor/github.com/elastic/
+	mkdir -p vendor/github.com/elastic/beats
+	git archive --remote ${BEAT_GOPATH}/src/github.com/elastic/beats HEAD | tar -x --exclude=x-pack -C vendor/github.com/elastic/beats
 	ln -sf ${PWD}/vendor/github.com/elastic/beats/metricbeat/scripts/generate_imports_helper.py ${PWD}/vendor/github.com/elastic/beats/script/generate_imports_helper.py
-	rm -rf vendor/github.com/elastic/beats/.git vendor/github.com/elastic/beats/x-pack
 	mkdir -p vendor/github.com/magefile
 	cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats/vendor/github.com/magefile/mage vendor/github.com/magefile
+	cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats/vendor/github.com/pkg vendor/github.com/
 
 .PHONY: git-init
 git-init:

generator/metricbeat/{beat}/_meta/docker.yml

@@ -0,0 +1,4 @@
+{{.BeatName}}.config.modules:
+  path: ${path.config}/modules.d/*.yml
+  reload.enabled: false
+

generator/metricbeat/{beat}/_meta/reference.yml

@@ -0,0 +1,54 @@
+########################## {{.BeatName}} Configuration ###########################
+
+# This file is a full configuration example documenting all non-deprecated
+# options in comments. For a shorter configuration example, that contains only
+# the most common options, please see metricbeat.yml in the same directory.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/metricbeat/index.html
+
+#============================  Config Reloading ===============================
+
+# Config reloading allows to dynamically load modules. Each file which is
+# monitored must contain one or multiple modules as a list.
+{{.BeatName}}.config.modules:
+
+  # Glob pattern for configuration reloading
+  path: ${path.config}/modules.d/*.yml
+
+  # Period on which files under path should be checked for changes
+  reload.period: 10s
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+# Maximum amount of time to randomly delay the start of a metricset. Use 0 to
+# disable startup delay.
+{{.BeatName}}.max_start_delay: 10s
+
+#============================== Autodiscover ===================================
+
+# Autodiscover allows you to detect changes in the system and spawn new modules
+# as they happen.
+
+#{{.BeatName}}.autodiscover:
+  # List of enabled autodiscover providers
+#  providers:
+#    - type: docker
+#      templates:
+#        - condition:
+#            equals.docker.container.image: etcd
+#          config:
+#            - module: etcd
+#              metricsets: ["leader", "self", "store"]
+#              period: 10s
+#              hosts: ["${host}:2379"]
+
+#=========================== Timeseries instance ===============================
+
+# Enabling this will add a `timeseries.instance` keyword field to all metric
+# events. For a given metricset, this field will be unique for every single item
+# being monitored.
+# This setting is experimental.
+
+#timeseries.enabled: false

generator/metricbeat/{beat}/_meta/short.yml

@@ -0,0 +1,19 @@
+
+#==========================  Modules configuration ============================
+
+{{.BeatName}}.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 1
+  index.codec: best_compression
+  #_source.enabled: false

generator/metricbeat/{beat}/cmd/modules.go

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/cfgfile"
+	"github.com/elastic/beats/libbeat/cmd"
+)
+
+// BuildModulesManager adds support for modules management to a beat
+func BuildModulesManager(beat *beat.Beat) (cmd.ModulesManager, error) {
+	config := beat.BeatConfig
+
+	glob, err := config.String("config.modules.path", -1)
+	if err != nil {
+		return nil, errors.Errorf("modules management requires '%s.config.modules.path' setting", Name)
+	}
+
+	if !strings.HasSuffix(glob, "*.yml") {
+		return nil, errors.Errorf("wrong settings for config.modules.path, it is expected to end with *.yml. Got: %s", glob)
+	}
+
+	modulesManager, err := cfgfile.NewGlobManager(glob, ".yml", ".disabled")
+	if err != nil {
+		return nil, errors.Wrap(err, "initialization error")
+	}
+	return modulesManager, nil
+}

generator/metricbeat/{beat}/cmd/root.go.tmpl

@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+	cmd "github.com/elastic/beats/libbeat/cmd"
+	"github.com/elastic/beats/libbeat/cmd/instance"
+	"github.com/elastic/beats/metricbeat/beater"
+	"github.com/elastic/beats/metricbeat/cmd/test"
+	"github.com/elastic/beats/metricbeat/mb/module"
+)
+
+// Name of this beat
+var Name = "{beat}"
+
+// RootCmd to handle beats cli
+var RootCmd *cmd.BeatsRootCmd
+
+var (
+	// Use a customized instance of Metricbeat where startup delay has
+	// been disabled to workaround the fact that Modules() will return
+	// the static modules (not the dynamic ones) with a start delay.
+	testModulesCreator = beater.Creator(
+		beater.WithModuleOptions(
+			module.WithMetricSetInfo(),
+			module.WithMaxStartDelay(0),
+		),
+	)
+)
+
+func init() {
+	RootCmd = cmd.GenRootCmdWithSettings(beater.DefaultCreator(), instance.Settings{Name: Name})
+	RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", BuildModulesManager))
+	RootCmd.TestCmd.AddCommand(test.GenTestModulesCmd(Name, "", testModulesCreator))
+}

generator/metricbeat/{beat}/magefile.go

@@ -134,5 +134,11 @@ func GoTestIntegration(ctx context.Context) error {
 
 // Config generates both the short/reference/docker configs.
 func Config() error {
-	return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".")
+	customDeps := devtools.ConfigFileParams{
+		ShortParts:     []string{"_meta/short.yml", devtools.LibbeatDir("_meta/config.yml.tmpl")},
+		ReferenceParts: []string{"_meta/reference.yml", devtools.LibbeatDir("_meta/config.reference.yml.tmpl")},
+		DockerParts:    []string{"_meta/docker.yml", devtools.LibbeatDir("_meta/config.docker.yml")},
+		ExtraVars:      map[string]interface{}{"BeatName": devtools.BeatName},
+	}
+	return devtools.Config(devtools.AllConfigTypes, customDeps, ".")
 }

generator/metricbeat/{beat}/main.go.tmpl

@@ -3,22 +3,15 @@ package main
 import (
 	"os"
 
-	"github.com/elastic/beats/libbeat/cmd/instance"
-	"github.com/elastic/beats/metricbeat/beater"
-
-	// Comment out the following line to exclude all official metricbeat modules and metricsets
-	_ "github.com/elastic/beats/metricbeat/include"
+	"{beat_path}/cmd"
 
 	// Make sure all your modules and metricsets are linked in this file
 	_ "{beat_path}/include"
 )
 
-var settings = instance.Settings{
-	Name: "{beat}",
-}
 
 func main() {
-	if err := instance.Run(settings, beater.DefaultCreator()); err != nil {
+	if err := cmd.RootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}
 }