From f2b4d8b52829a440b712d27ad6c0f207349a5cf2 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 11 Sep 2019 13:14:46 -0700
Subject: [PATCH 01/34] WiP: converting generator.py to mage

---
 generator/common/beatgen/beatgen.go | 171 ++++++++++++++++++++++++++++
 magefile.go                         |   6 +
 2 files changed, 177 insertions(+)
 create mode 100644 generator/common/beatgen/beatgen.go

diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
new file mode 100644
index 00000000000..d12c323dda1
--- /dev/null
+++ b/generator/common/beatgen/beatgen.go
@@ -0,0 +1,171 @@
+package beatgen
+
+import (
+	"bufio"
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/pkg/errors"
+)
+
+// ConfigItem represents a value that must be configured for the custom beat
+type ConfigItem struct {
+	Key     string
+	Default func(map[string]string) string
+}
+
+// required user config for a custom beat
+// These are specified in env variables with newbeat_*
+var configList = []ConfigItem{
+	{
+		Key: "project_name",
+		Default: func(cfg map[string]string) string {
+			return "examplebeat"
+		},
+	},
+	{
+		Key: "github_name",
+		Default: func(cfg map[string]string) string {
+			return "your-github-name"
+		},
+	},
+	{
+		Key: "beat_path",
+		Default: func(cfg map[string]string) string {
+			ghName, _ := cfg["github_name"]
+			beatName, _ := cfg["project_name"]
+			return "github.com/" + ghName + "/" + strings.ToLower(beatName)
+		},
+	},
+	{
+		Key: "full_name",
+		Default: func(cfg map[string]string) string {
+			return "Firstname Lastname"
+		},
+	},
+	{
+		Key: "type",
+		Default: func(cfg map[string]string) string {
+			return "beat"
+		},
+	},
+}
+
+var cfgPrefix = "NEWBEAT"
+
+// Generate generates a new custom beat
+func Generate() error {
+	cfg, err := getConfig()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%#v\n", cfg)
+	return genNewBeat(cfg)
+}
+
+// genNewBeat generates a new custom beat
+func genNewBeat(config map[string]string) error {
+
+	genPath := devtools.OSSBeatDir("generator", config["type"], "{beat}")
+	fmt.Printf("Attempting to read from gen directory %s\n", genPath)
+	err := filepath.Walk(genPath, func(path string, info os.FileInfo, err error) error {
+		//fmt.Printf("Got path %s, Fileinfo: %v Name: %s\n", path, info.IsDir(), info.Name())
+		newBase := filepath.Join(build.Default.GOPATH, "src", config["beat_path"])
+		replacePath := strings.Replace(path, genPath, newBase, -1)
+		//fmt.Printf("replacing with path %s\n", replacePath)
+
+		writePath := strings.Replace(replacePath, "{beat}", config["project_name"], -1)
+		writePath = strings.Replace(writePath, ".go.tmpl", ".go", -1)
+		if info.IsDir() {
+			err := os.MkdirAll(writePath, 0755)
+			if err != nil {
+				return errors.Wrap(err, "error creating directory")
+			}
+		} else {
+
+			//dump original source file
+			tmplFile, err := ioutil.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			newFile := replaceVars(config, string(tmplFile))
+
+			fmt.Printf("Attempting to write to %s\n", writePath)
+			err = ioutil.WriteFile(writePath, []byte(newFile), 0644)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+// replaceVars replaces any template vars in a target file
+// We're not using the golang template engine as it seems a tad heavy-handed for this use case
+// We have a dozen or so files across various languages (go, make, etc) and most just need one or two vars replaced.
+func replaceVars(config map[string]string, fileBody string) string {
+	var newBody = fileBody
+	for tmplName, tmplValue := range config {
+		tmplStr := fmt.Sprintf("{%s}", tmplName)
+		newBody = strings.ReplaceAll(newBody, tmplStr, tmplValue)
+	}
+
+	return newBody
+}
+
+// returns a "compleated" config object with everything we need
+func getConfig() (map[string]string, error) {
+
+	userCfg := make(map[string]string)
+	for _, cfgVal := range configList {
+		var cfgKey string
+		var err error
+		cfgKey, isSet := getEnvConfig(cfgVal.Key)
+		if !isSet {
+			cfgKey, err = getValFromUser(cfgVal.Key, cfgVal.Default(userCfg))
+			if err != nil {
+				return userCfg, err
+			}
+		}
+		userCfg[cfgVal.Key] = cfgKey
+	}
+
+	return userCfg, nil
+
+}
+
+func getEnvConfig(cfgKey string) (string, bool) {
+	EnvKey := fmt.Sprintf("%s_%s", cfgPrefix, strings.ToUpper(cfgKey))
+
+	envKey := os.Getenv(EnvKey)
+
+	if envKey == "" {
+		return envKey, false
+	}
+	return envKey, true
+}
+
+// getValFromUser returns a config object from the user. If they don't enter one, fallback to the default
+func getValFromUser(cfgKey, def string) (string, error) {
+	reader := bufio.NewReader(os.Stdin)
+
+	// Human-readable prompt
+	fmt.Printf("Enter a %s [%s]: ", strings.Replace(cfgKey, "_", " ", -1), def)
+	str, err := reader.ReadString('\n')
+	if err != nil {
+		return "", err
+	}
+	if str == "\n" {
+		return def, nil
+	}
+	return strings.TrimSpace(str), nil
+
+}
diff --git a/magefile.go b/magefile.go
index 25eddd0acef..7ad360a49ca 100644
--- a/magefile.go
+++ b/magefile.go
@@ -24,6 +24,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/elastic/beats/generator/common/beatgen"
 	"github.com/magefile/mage/mg"
 	"github.com/pkg/errors"
 	"go.uber.org/multierr"
@@ -45,6 +46,11 @@ var (
 	}
 )
 
+// GenerateCustomBeat generates a new custom beat
+func GenerateCustomBeat() error {
+	return beatgen.Generate()
+}
+
 // PackageBeatDashboards packages the dashboards from all Beats into a zip
 // file. The dashboards must be generated first.
 func PackageBeatDashboards() error {

From def9e618a374c991537026a52c2c9ff0ea38f12b Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 11 Sep 2019 14:54:05 -0700
Subject: [PATCH 02/34] quick cleanup, add type check

---
 generator/common/beatgen/beatgen.go | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index d12c323dda1..ff0f87ce0a9 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package beatgen
 
 import (
@@ -64,20 +81,20 @@ func Generate() error {
 	if err != nil {
 		return err
 	}
-	fmt.Printf("%#v\n", cfg)
 	return genNewBeat(cfg)
 }
 
 // genNewBeat generates a new custom beat
+// We assume our config object is populated and valid here
 func genNewBeat(config map[string]string) error {
+	if config["type"] != "beat" && config["type"] != "metricbeat" {
+		return fmt.Errorf("%s is not a valid custom beat type. Valid types are 'beat' and 'metricbeat'", config["type"])
+	}
 
 	genPath := devtools.OSSBeatDir("generator", config["type"], "{beat}")
-	fmt.Printf("Attempting to read from gen directory %s\n", genPath)
 	err := filepath.Walk(genPath, func(path string, info os.FileInfo, err error) error {
-		//fmt.Printf("Got path %s, Fileinfo: %v Name: %s\n", path, info.IsDir(), info.Name())
 		newBase := filepath.Join(build.Default.GOPATH, "src", config["beat_path"])
 		replacePath := strings.Replace(path, genPath, newBase, -1)
-		//fmt.Printf("replacing with path %s\n", replacePath)
 
 		writePath := strings.Replace(replacePath, "{beat}", config["project_name"], -1)
 		writePath = strings.Replace(writePath, ".go.tmpl", ".go", -1)
@@ -95,7 +112,6 @@ func genNewBeat(config map[string]string) error {
 			}
 			newFile := replaceVars(config, string(tmplFile))
 
-			fmt.Printf("Attempting to write to %s\n", writePath)
 			err = ioutil.WriteFile(writePath, []byte(newFile), 0644)
 			if err != nil {
 				return err
@@ -123,7 +139,6 @@ func replaceVars(config map[string]string, fileBody string) string {
 
 // returns a "compleated" config object with everything we need
 func getConfig() (map[string]string, error) {
-
 	userCfg := make(map[string]string)
 	for _, cfgVal := range configList {
 		var cfgKey string

From 454036804b00a01815aff8a7de05b24ed51d6ffd Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Mon, 16 Sep 2019 10:18:00 -0700
Subject: [PATCH 03/34] more work on moving generator setup

---
 generator/beat/{beat}/beater/{beat}.go | 10 +--
 generator/common/beatgen/beatgen.go    | 68 +++++-------------
 generator/common/beatgen/creator.go    | 67 ++++++++++++++++++
 generator/common/beatgen/setup.go      | 97 ++++++++++++++++++++++++++
 4 files changed, 188 insertions(+), 54 deletions(-)
 create mode 100644 generator/common/beatgen/creator.go
 create mode 100644 generator/common/beatgen/setup.go

diff --git a/generator/beat/{beat}/beater/{beat}.go b/generator/beat/{beat}/beater/{beat}.go
index 7c0a1027df2..8031896402b 100644
--- a/generator/beat/{beat}/beater/{beat}.go
+++ b/generator/beat/{beat}/beater/{beat}.go
@@ -11,8 +11,8 @@ import (
 	"{beat_path}/config"
 )
 
-// {Beat} configuration.
-type {Beat} struct {
+// {beat} configuration.
+type {beat} struct {
 	done   chan struct{}
 	config config.Config
 	client beat.Client
@@ -25,7 +25,7 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
 		return nil, fmt.Errorf("Error reading config file: %v", err)
 	}
 
-	bt := &{Beat}{
+	bt := &{beat}{
 		done:   make(chan struct{}),
 		config: c,
 	}
@@ -33,7 +33,7 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
 }
 
 // Run starts {beat}.
-func (bt *{Beat}) Run(b *beat.Beat) error {
+func (bt *{beat}) Run(b *beat.Beat) error {
 	logp.Info("{beat} is running! Hit CTRL-C to stop it.")
 
 	var err error
@@ -65,7 +65,7 @@ func (bt *{Beat}) Run(b *beat.Beat) error {
 }
 
 // Stop stops {beat}.
-func (bt *{Beat}) Stop() {
+func (bt *{beat}) Stop() {
 	bt.client.Close()
 	close(bt.done)
 }
diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index ff0f87ce0a9..17d57353954 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -21,12 +21,12 @@ import (
 	"bufio"
 	"fmt"
 	"go/build"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
-	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
 )
 
@@ -81,60 +81,30 @@ func Generate() error {
 	if err != nil {
 		return err
 	}
-	return genNewBeat(cfg)
-}
-
-// genNewBeat generates a new custom beat
-// We assume our config object is populated and valid here
-func genNewBeat(config map[string]string) error {
-	if config["type"] != "beat" && config["type"] != "metricbeat" {
-		return fmt.Errorf("%s is not a valid custom beat type. Valid types are 'beat' and 'metricbeat'", config["type"])
+	err = genNewBeat(cfg)
+	if err != nil {
+		return err
 	}
 
-	genPath := devtools.OSSBeatDir("generator", config["type"], "{beat}")
-	err := filepath.Walk(genPath, func(path string, info os.FileInfo, err error) error {
-		newBase := filepath.Join(build.Default.GOPATH, "src", config["beat_path"])
-		replacePath := strings.Replace(path, genPath, newBase, -1)
+	err = os.Chdir(filepath.Join(build.Default.GOPATH, "src", cfg["beat_path"]))
+	if err != nil {
+		return err
+	}
 
-		writePath := strings.Replace(replacePath, "{beat}", config["project_name"], -1)
-		writePath = strings.Replace(writePath, ".go.tmpl", ".go", -1)
-		if info.IsDir() {
-			err := os.MkdirAll(writePath, 0755)
-			if err != nil {
-				return errors.Wrap(err, "error creating directory")
-			}
-		} else {
+	mg.Deps(CopyVendor)
+	mg.Deps(RunSetup)
+	mg.Deps(GitInit)
 
-			//dump original source file
-			tmplFile, err := ioutil.ReadFile(path)
-			if err != nil {
-				return err
-			}
-			newFile := replaceVars(config, string(tmplFile))
-
-			err = ioutil.WriteFile(writePath, []byte(newFile), 0644)
-			if err != nil {
-				return err
-			}
+	if cfg["type"] == "metricbeat" {
+		err = sh.Run("make", "create-metricset")
+		if err != nil {
+			return errors.Wrap(err, "error running create-metricset")
 		}
-
-		return nil
-	})
-
-	return err
-}
-
-// replaceVars replaces any template vars in a target file
-// We're not using the golang template engine as it seems a tad heavy-handed for this use case
-// We have a dozen or so files across various languages (go, make, etc) and most just need one or two vars replaced.
-func replaceVars(config map[string]string, fileBody string) string {
-	var newBody = fileBody
-	for tmplName, tmplValue := range config {
-		tmplStr := fmt.Sprintf("{%s}", tmplName)
-		newBody = strings.ReplaceAll(newBody, tmplStr, tmplValue)
 	}
 
-	return newBody
+	mg.Deps(GitAdd)
+
+	return nil
 }
 
 // returns a "compleated" config object with everything we need
diff --git a/generator/common/beatgen/creator.go b/generator/common/beatgen/creator.go
new file mode 100644
index 00000000000..8aca4925ec3
--- /dev/null
+++ b/generator/common/beatgen/creator.go
@@ -0,0 +1,67 @@
+package beatgen
+
+import (
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/pkg/errors"
+)
+
+// genNewBeat generates a new custom beat
+// We assume our config object is populated and valid here
+func genNewBeat(config map[string]string) error {
+	if config["type"] != "beat" && config["type"] != "metricbeat" {
+		return fmt.Errorf("%s is not a valid custom beat type. Valid types are 'beat' and 'metricbeat'", config["type"])
+	}
+
+	genPath := devtools.OSSBeatDir("generator", config["type"], "{beat}")
+	err := filepath.Walk(genPath, func(path string, info os.FileInfo, err error) error {
+		newBase := filepath.Join(build.Default.GOPATH, "src", config["beat_path"])
+		replacePath := strings.Replace(path, genPath, newBase, -1)
+
+		writePath := strings.Replace(replacePath, "{beat}", config["project_name"], -1)
+		writePath = strings.Replace(writePath, ".go.tmpl", ".go", -1)
+		if info.IsDir() {
+			err := os.MkdirAll(writePath, 0755)
+			if err != nil {
+				return errors.Wrap(err, "error creating directory")
+			}
+		} else {
+
+			//dump original source file
+			tmplFile, err := ioutil.ReadFile(path)
+			if err != nil {
+				return err
+			}
+			newFile := replaceVars(config, string(tmplFile))
+
+			err = ioutil.WriteFile(writePath, []byte(newFile), 0644)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+
+	return err
+}
+
+// replaceVars replaces any template vars in a target file
+// We're not using the golang template engine as it seems a tad heavy-handed for this use case
+// We have a dozen or so files across various languages (go, make, etc) and most just need one or two vars replaced.
+func replaceVars(config map[string]string, fileBody string) string {
+	var newBody = fileBody
+	config["beat"] = strings.ToLower(config["project_name"])
+	for tmplName, tmplValue := range config {
+		tmplStr := fmt.Sprintf("{%s}", tmplName)
+		newBody = strings.ReplaceAll(newBody, tmplStr, tmplValue)
+	}
+
+	return newBody
+}
diff --git a/generator/common/beatgen/setup.go b/generator/common/beatgen/setup.go
new file mode 100644
index 00000000000..fb3e77d0b7f
--- /dev/null
+++ b/generator/common/beatgen/setup.go
@@ -0,0 +1,97 @@
+package beatgen
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
+)
+
+// RunSetup runs any remaining setup commands after the vendor directory has been setup
+func RunSetup() error {
+	vendorPath := "./vendor/github.com/"
+
+	//Copy mage stuff
+	err := os.MkdirAll(filepath.Join(vendorPath, "magefile"), 0755)
+	if err != nil {
+		return errors.Wrap(err, "error making mage dir")
+	}
+
+	err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/magefile/mage"), filepath.Join(vendorPath, "magefile"))
+	if err != nil {
+		return errors.Wrap(err, "error copying vendored magefile")
+	}
+
+	//Copy the pkg helper
+	err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/pkg"), vendorPath)
+	if err != nil {
+		return errors.Wrap(err, "error copying pkg to vendor")
+	}
+
+	return sh.Run("ln", "-sf", filepath.Join(vendorPath, "elastic/beats/metricbeat/scripts/generate_imports_helper.py"))
+
+}
+
+// CopyVendor copies a new version of beats into the vendor directory of PWD
+// By default this uses git archive, meaning any uncommitted changes will not be copied.
+// Set the NEWBEAT_DEV env variable to use a slow `cp` copy that will catch uncommited changes
+func CopyVendor() error {
+	vendorPath := "./vendor/github.com/elastic/"
+	beatPath, err := devtools.ElasticBeatsDir()
+	if err != nil {
+		return errors.Wrap(err, "Could not find ElasticBeatsDir")
+	}
+	err = os.MkdirAll(vendorPath, 0755)
+	if err != nil {
+		return errors.Wrap(err, "error creating vendor dir")
+	}
+	if _, isDev := os.LookupEnv(cfgPrefix + "_DEV"); isDev {
+		//Dev mode. Use CP.
+		fmt.Printf("CopyVendor unning in dev mode.\n")
+		err = sh.Run("cp", "-R", beatPath, vendorPath)
+		if err != nil {
+			return errors.Wrap(err, "error copying vendor dir")
+		}
+		err = sh.Rm(filepath.Join(vendorPath, ".git"))
+		if err != nil {
+			return errors.Wrap(err, "error removing vendor git directory")
+		}
+		err = sh.Rm(filepath.Join(vendorPath, "x-pack"))
+		if err != nil {
+			return errors.Wrap(err, "error removing x-pack directory")
+		}
+	} else {
+		//not dev mode. Use git archive
+		vendorPath = filepath.Join(vendorPath, "beats")
+		err = os.MkdirAll(vendorPath, 0755)
+		if err != nil {
+			return errors.Wrap(err, "error creating vendor dir")
+		}
+		err = sh.Run("sh",
+			"-c",
+			"git archive --remote "+beatPath+" HEAD |  tar -x --exclude=x-pack -C "+vendorPath)
+		if err != nil {
+			return errors.Wrap(err, "error running git archive")
+		}
+	}
+
+	return nil
+
+}
+
+// GitInit initializes a new git repo in the current directory
+func GitInit() error {
+	return sh.Run("git", "init")
+}
+
+// GitAdd adds the current working directory to git and does an initial commit
+func GitAdd() error {
+	err := sh.Run("git", "add", "-A")
+	if err != nil {
+		return errors.Wrap(err, "error running git add")
+	}
+	return sh.Run("git", "commit", "-q", "-m", "Initial commit, Add generated files")
+}

From 1d9e6ae5ee30ce84ad5355a47e2e9c5694826086 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 17 Sep 2019 09:44:02 -0700
Subject: [PATCH 04/34] port setup over to mage

---
 generator/common/beatgen/beatgen.go |  3 ++-
 generator/common/beatgen/setup.go   | 12 +++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index 17d57353954..60dd92ed628 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -96,12 +96,13 @@ func Generate() error {
 	mg.Deps(GitInit)
 
 	if cfg["type"] == "metricbeat" {
-		err = sh.Run("make", "create-metricset")
+		err = sh.RunV("make", "create-metricset")
 		if err != nil {
 			return errors.Wrap(err, "error running create-metricset")
 		}
 	}
 
+	mg.Deps(Update)
 	mg.Deps(GitAdd)
 
 	return nil
diff --git a/generator/common/beatgen/setup.go b/generator/common/beatgen/setup.go
index fb3e77d0b7f..a4237e4799c 100644
--- a/generator/common/beatgen/setup.go
+++ b/generator/common/beatgen/setup.go
@@ -31,7 +31,12 @@ func RunSetup() error {
 		return errors.Wrap(err, "error copying pkg to vendor")
 	}
 
-	return sh.Run("ln", "-sf", filepath.Join(vendorPath, "elastic/beats/metricbeat/scripts/generate_imports_helper.py"))
+	pwd, err := os.Getwd()
+	if err != nil {
+		return errors.Wrap(err, "error gettting current directory")
+	}
+
+	return sh.Run("ln", "-sf", filepath.Join(pwd, vendorPath, "elastic/beats/metricbeat/scripts/generate_imports_helper.py"), filepath.Join(pwd, vendorPath, "elastic/beats/script/generate_imports_helper.py"))
 
 }
 
@@ -95,3 +100,8 @@ func GitAdd() error {
 	}
 	return sh.Run("git", "commit", "-q", "-m", "Initial commit, Add generated files")
 }
+
+// Update updates the generated files (aka make update).
+func Update() error {
+	return sh.Run("make", "update")
+}

From 7dd3d387abecbd06c9b4aa1b6dc25bf78db1736b Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 17 Sep 2019 14:09:45 -0700
Subject: [PATCH 05/34] more work on copy vendor, clean up

---
 dev-tools/mage/settings.go                    |  8 ++++
 generator/common/beatgen/beatgen.go           | 48 +++++++++++++++----
 .../common/beatgen/{ => setup}/creator.go     |  9 ++--
 generator/common/beatgen/{ => setup}/setup.go | 12 +++--
 generator/metricbeat/{beat}/magefile.go       |  6 +++
 heartbeat/magefile.go                         |  6 +++
 6 files changed, 73 insertions(+), 16 deletions(-)
 rename generator/common/beatgen/{ => setup}/creator.go (90%)
 rename generator/common/beatgen/{ => setup}/setup.go (91%)

diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go
index 07db69cca2c..dfe67432607 100644
--- a/dev-tools/mage/settings.go
+++ b/dev-tools/mage/settings.go
@@ -235,6 +235,14 @@ var (
 	elasticBeatsDirLock  sync.Mutex
 )
 
+// SetElasticBeatsDir sets the internal elastic beats dir to a preassigned value
+func SetElasticBeatsDir(path string) {
+	elasticBeatsDirLock.Lock()
+	defer elasticBeatsDirLock.Unlock()
+
+	elasticBeatsDirValue = path
+}
+
 // ElasticBeatsDir returns the path to Elastic beats dir.
 func ElasticBeatsDir() (string, error) {
 	elasticBeatsDirLock.Lock()
diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index 60dd92ed628..82a662ddd26 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -25,6 +25,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/elastic/beats/generator/common/beatgen/setup"
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
@@ -73,38 +75,52 @@ var configList = []ConfigItem{
 	},
 }
 
-var cfgPrefix = "NEWBEAT"
-
 // Generate generates a new custom beat
 func Generate() error {
 	cfg, err := getConfig()
 	if err != nil {
 		return err
 	}
-	err = genNewBeat(cfg)
+	err = setup.GenNewBeat(cfg)
 	if err != nil {
 		return err
 	}
 
-	err = os.Chdir(filepath.Join(build.Default.GOPATH, "src", cfg["beat_path"]))
+	absBeatPath := filepath.Join(build.Default.GOPATH, "src", cfg["beat_path"])
+
+	err = os.Chdir(absBeatPath)
 	if err != nil {
 		return err
 	}
 
-	mg.Deps(CopyVendor)
-	mg.Deps(RunSetup)
-	mg.Deps(GitInit)
+	mg.Deps(setup.CopyVendor)
+	mg.Deps(setup.RunSetup)
+	mg.Deps(setup.GitInit)
 
 	if cfg["type"] == "metricbeat" {
+		//This is runV because it'll ask for user input, so we need stdout.
 		err = sh.RunV("make", "create-metricset")
 		if err != nil {
 			return errors.Wrap(err, "error running create-metricset")
 		}
 	}
 
-	mg.Deps(Update)
-	mg.Deps(GitAdd)
+	//mg.Deps(setup.Update)
+	mg.Deps(setup.GitAdd)
+
+	return nil
+}
+
+// VendorUpdate updates the beat vendor directory
+func VendorUpdate() error {
+
+	err := sh.Rm("./vendor/github.com/elastic/beats")
+	if err != nil {
+		return errors.Wrap(err, "error removing vendor dir")
+	}
 
+	devtools.SetElasticBeatsDir(getAbsoluteBeatsPath())
+	mg.Deps(setup.CopyVendor)
 	return nil
 }
 
@@ -129,7 +145,7 @@ func getConfig() (map[string]string, error) {
 }
 
 func getEnvConfig(cfgKey string) (string, bool) {
-	EnvKey := fmt.Sprintf("%s_%s", cfgPrefix, strings.ToUpper(cfgKey))
+	EnvKey := fmt.Sprintf("%s_%s", setup.CfgPrefix, strings.ToUpper(cfgKey))
 
 	envKey := os.Getenv(EnvKey)
 
@@ -155,3 +171,15 @@ func getValFromUser(cfgKey, def string) (string, error) {
 	return strings.TrimSpace(str), nil
 
 }
+
+// getAbsoluteBeatsPath tries to infer the "real" non-vendor beats path
+func getAbsoluteBeatsPath() string {
+
+	beatsImportPath := "github.com/elastic/beats"
+	gopath := os.Getenv("GOPATH")
+	if gopath != "" {
+		return filepath.Join(gopath, "src", beatsImportPath)
+	}
+	return filepath.Join(build.Default.GOPATH, "src", beatsImportPath)
+
+}
diff --git a/generator/common/beatgen/creator.go b/generator/common/beatgen/setup/creator.go
similarity index 90%
rename from generator/common/beatgen/creator.go
rename to generator/common/beatgen/setup/creator.go
index 8aca4925ec3..0dd3d4a314d 100644
--- a/generator/common/beatgen/creator.go
+++ b/generator/common/beatgen/setup/creator.go
@@ -1,4 +1,4 @@
-package beatgen
+package setup
 
 import (
 	"fmt"
@@ -12,9 +12,12 @@ import (
 	"github.com/pkg/errors"
 )
 
-// genNewBeat generates a new custom beat
+// CfgPrefix specifies the env variable prefix used to configure the beat
+var CfgPrefix = "NEWBEAT"
+
+// GenNewBeat generates a new custom beat
 // We assume our config object is populated and valid here
-func genNewBeat(config map[string]string) error {
+func GenNewBeat(config map[string]string) error {
 	if config["type"] != "beat" && config["type"] != "metricbeat" {
 		return fmt.Errorf("%s is not a valid custom beat type. Valid types are 'beat' and 'metricbeat'", config["type"])
 	}
diff --git a/generator/common/beatgen/setup.go b/generator/common/beatgen/setup/setup.go
similarity index 91%
rename from generator/common/beatgen/setup.go
rename to generator/common/beatgen/setup/setup.go
index a4237e4799c..739aa59e403 100644
--- a/generator/common/beatgen/setup.go
+++ b/generator/common/beatgen/setup/setup.go
@@ -1,4 +1,4 @@
-package beatgen
+package setup
 
 import (
 	"fmt"
@@ -53,9 +53,10 @@ func CopyVendor() error {
 	if err != nil {
 		return errors.Wrap(err, "error creating vendor dir")
 	}
-	if _, isDev := os.LookupEnv(cfgPrefix + "_DEV"); isDev {
+	if _, isDev := os.LookupEnv(CfgPrefix + "_DEV"); isDev {
 		//Dev mode. Use CP.
-		fmt.Printf("CopyVendor unning in dev mode.\n")
+		fmt.Printf("CopyVendor running in dev mode, elastic/beats will be copied into the vendor directory with cp\n")
+		vendorPath = filepath.Join(vendorPath, "beats")
 		err = sh.Run("cp", "-R", beatPath, vendorPath)
 		if err != nil {
 			return errors.Wrap(err, "error copying vendor dir")
@@ -87,6 +88,11 @@ func CopyVendor() error {
 
 }
 
+// getRealBeatsRepo gets a non
+func getRealBeatsRepo() {
+
+}
+
 // GitInit initializes a new git repo in the current directory
 func GitInit() error {
 	return sh.Run("git", "init")
diff --git a/generator/metricbeat/{beat}/magefile.go b/generator/metricbeat/{beat}/magefile.go
index dc575b5da9d..db9ac8970c0 100644
--- a/generator/metricbeat/{beat}/magefile.go
+++ b/generator/metricbeat/{beat}/magefile.go
@@ -9,6 +9,7 @@ import (
 	"github.com/magefile/mage/mg"
 
 	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/elastic/beats/generator/common/beatgen"
 	metricbeat "github.com/elastic/beats/metricbeat/scripts/mage"
 
 	// mage:import
@@ -36,6 +37,11 @@ func init() {
 	devtools.BeatVendor = "{full_name}"
 }
 
+// VendorUpdate updates elastic/beats in the vendor dir
+func VendorUpdate() error {
+	return beatgen.VendorUpdate()
+}
+
 // CollectAll generates the docs and the fields.
 func CollectAll() {
 	mg.Deps(collectors.CollectDocs, FieldsDocs)
diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go
index 3119eacf656..7aafc45cf23 100644
--- a/heartbeat/magefile.go
+++ b/heartbeat/magefile.go
@@ -28,6 +28,7 @@ import (
 	"github.com/magefile/mage/sh"
 
 	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/elastic/beats/generator/common/beatgen"
 	heartbeat "github.com/elastic/beats/heartbeat/scripts/mage"
 
 	// mage:import
@@ -42,6 +43,11 @@ func init() {
 	devtools.BeatServiceName = "heartbeat-elastic"
 }
 
+// VendorUpdate updates elastic/beats in the vendor dir
+func VendorUpdate() error {
+	return beatgen.VendorUpdate()
+}
+
 // Build builds the Beat binary.
 func Build() error {
 	return devtools.Build(devtools.DefaultBuildArgs())

From a2985d3ac4ede07472b05d5553f68fc4f86d6f94 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 17 Sep 2019 14:11:41 -0700
Subject: [PATCH 06/34] add lic headers

---
 generator/common/beatgen/setup/creator.go | 17 +++++++++++++++++
 generator/common/beatgen/setup/setup.go   | 18 ++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/generator/common/beatgen/setup/creator.go b/generator/common/beatgen/setup/creator.go
index 0dd3d4a314d..e10b9d17237 100644
--- a/generator/common/beatgen/setup/creator.go
+++ b/generator/common/beatgen/setup/creator.go
@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package setup
 
 import (
diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go
index 739aa59e403..50504e84593 100644
--- a/generator/common/beatgen/setup/setup.go
+++ b/generator/common/beatgen/setup/setup.go
@@ -1,3 +1,20 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 package setup
 
 import (
@@ -57,6 +74,7 @@ func CopyVendor() error {
 		//Dev mode. Use CP.
 		fmt.Printf("CopyVendor running in dev mode, elastic/beats will be copied into the vendor directory with cp\n")
 		vendorPath = filepath.Join(vendorPath, "beats")
+
 		err = sh.Run("cp", "-R", beatPath, vendorPath)
 		if err != nil {
 			return errors.Wrap(err, "error copying vendor dir")

From 140e032051a31e5ab1dc01170cc938783f42a625 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 18 Sep 2019 12:38:00 -0700
Subject: [PATCH 07/34] update and clean up beat template files

---
 generator/beat/{beat}/magefile.go | 56 -------------------------------
 1 file changed, 56 deletions(-)
 delete mode 100644 generator/beat/{beat}/magefile.go

diff --git a/generator/beat/{beat}/magefile.go b/generator/beat/{beat}/magefile.go
deleted file mode 100644
index bd67f87baad..00000000000
--- a/generator/beat/{beat}/magefile.go
+++ /dev/null
@@ -1,56 +0,0 @@
-// +build mage
-
-package main
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/magefile/mage/mg"
-
-	devtools "github.com/elastic/beats/dev-tools/mage"
-
-	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/common"
-	// mage:import
-	"github.com/elastic/beats/dev-tools/mage/target/build"
-	// mage:import
-	"github.com/elastic/beats/dev-tools/mage/target/update"
-	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/test"
-	// mage:import
-	_ "github.com/elastic/beats/dev-tools/mage/target/unittest"
-	// mage:import
-	"github.com/elastic/beats/dev-tools/mage/target/pkg"
-)
-
-func init() {
-	devtools.SetBuildVariableSources(devtools.DefaultBeatBuildVariableSources)
-
-	devtools.BeatDescription = "One sentence description of the Beat."
-	devtools.BeatVendor = "{full_name}"
-}
-
-// Package packages the Beat for distribution.
-// Use SNAPSHOT=true to build snapshots.
-// Use PLATFORMS to control the target platforms.
-func Package() {
-	start := time.Now()
-	defer func() { fmt.Println("package ran for", time.Since(start)) }()
-
-	devtools.UseCommunityBeatPackaging()
-
-	mg.Deps(update.Update)
-	mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon)
-	mg.SerialDeps(devtools.Package, pkg.PackageTest)
-}
-
-// Config generates both the short/reference/docker configs.
-func Config() error {
-	return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".")
-}
-
-//Fields generates a fields.yml for the Beat.
-func Fields() error {
-	return devtools.GenerateFieldsYAML()
-}

From 762061d44692f754001b0ead1791498db05561d1 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 18 Sep 2019 13:05:20 -0700
Subject: [PATCH 08/34] bugfixes

---
 generator/common/beatgen/beatgen.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index 82a662ddd26..501f8ccc180 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -105,7 +105,7 @@ func Generate() error {
 		}
 	}
 
-	//mg.Deps(setup.Update)
+	mg.Deps(setup.Update)
 	mg.Deps(setup.GitAdd)
 
 	return nil

From 517f1980230cc2f5957be9eb843b50e4882a9446 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 18 Sep 2019 13:48:26 -0700
Subject: [PATCH 09/34] rebase

---
 generator/beat/{beat}/magefile.go | 85 +++++++++++++++++++++++++++++++
 1 file changed, 85 insertions(+)
 create mode 100644 generator/beat/{beat}/magefile.go

diff --git a/generator/beat/{beat}/magefile.go b/generator/beat/{beat}/magefile.go
new file mode 100644
index 00000000000..82edcec94a3
--- /dev/null
+++ b/generator/beat/{beat}/magefile.go
@@ -0,0 +1,85 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build mage
+
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
+
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	"github.com/elastic/beats/generator/common/beatgen"
+
+	// mage:import
+	"github.com/elastic/beats/dev-tools/mage/target/pkg"
+	// mage:import
+	"github.com/elastic/beats/dev-tools/mage/target/build"
+	// mage:import
+	_ "github.com/elastic/beats/dev-tools/mage/target/common"
+	// mage:import
+	_ "github.com/elastic/beats/dev-tools/mage/target/test"
+	// mage:import
+	_ "github.com/elastic/beats/dev-tools/mage/target/unittest"
+	// mage:import
+	_ "github.com/elastic/beats/dev-tools/mage/target/integtest"
+)
+
+func init() {
+	devtools.SetBuildVariableSources(devtools.DefaultBeatBuildVariableSources)
+
+	devtools.BeatDescription = "One sentence description of the Beat."
+	devtools.BeatVendor = "{full_name}"
+}
+
+// VendorUpdate updates elastic/beats in the vendor dir
+func VendorUpdate() error {
+	return beatgen.VendorUpdate()
+}
+
+// Package packages the Beat for distribution.
+// Use SNAPSHOT=true to build snapshots.
+// Use PLATFORMS to control the target platforms.
+func Package() {
+	start := time.Now()
+	defer func() { fmt.Println("package ran for", time.Since(start)) }()
+
+	devtools.UseCommunityBeatPackaging()
+
+	mg.Deps(Update)
+	mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon)
+	mg.SerialDeps(devtools.Package, pkg.TestPackages)
+}
+
+// Update updates the generated files (aka make update).
+func Update() error {
+	return sh.Run("make", "update")
+}
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return devtools.GenerateFieldsYAML()
+}
+
+// Config generates both the short/reference/docker configs.
+func Config() error {
+	return devtools.Config(devtools.AllConfigTypes, devtools.ConfigFileParams{}, ".")
+}

From cf1a31ec6d1a536429fb7e1b0e9f7f7e0d688181 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 18 Sep 2019 13:52:40 -0700
Subject: [PATCH 10/34] exclude beat template dir

---
 magefile.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/magefile.go b/magefile.go
index 7ad360a49ca..e932e141249 100644
--- a/magefile.go
+++ b/magefile.go
@@ -135,6 +135,7 @@ func CheckLicenseHeaders() error {
 			licenser.Exclude("x-pack"),
 			licenser.Exclude("generator/beat/{beat}"),
 			licenser.Exclude("generator/metricbeat/{beat}"),
+			licenser.Exclude("generator/beat/{beat}"),
 		),
 		licenser(
 			licenser.Check(),

From 6e8fa6732ba40aeda44e8c2bf6d52ac1ecf9e405 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Thu, 19 Sep 2019 07:36:07 -0700
Subject: [PATCH 11/34] fix PackageTest

---
 generator/beat/{beat}/magefile.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generator/beat/{beat}/magefile.go b/generator/beat/{beat}/magefile.go
index 82edcec94a3..af1c10043cd 100644
--- a/generator/beat/{beat}/magefile.go
+++ b/generator/beat/{beat}/magefile.go
@@ -66,7 +66,7 @@ func Package() {
 
 	mg.Deps(Update)
 	mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon)
-	mg.SerialDeps(devtools.Package, pkg.TestPackages)
+	mg.SerialDeps(devtools.Package, pkg.PackageTest)
 }
 
 // Update updates the generated files (aka make update).

From 45a456c1ad4adf7ef67639d7e0c84db3df289be2 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Thu, 19 Sep 2019 11:23:46 -0700
Subject: [PATCH 12/34] work on porting test

---
 generator/common/Makefile           | 23 ++++++++++++-----------
 generator/metricbeat/{beat}/main.go |  1 -
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/generator/common/Makefile b/generator/common/Makefile
index e880b412e32..3c3be69f805 100644
--- a/generator/common/Makefile
+++ b/generator/common/Makefile
@@ -13,13 +13,10 @@ test: prepare-test
 	export GOPATH=${PWD}/build ; \
 	export PATH=$${GOPATH}/bin:${PATH}; \
 	cd ${BEAT_PATH} ; \
-	$(MAKE) copy-vendor || exit 1  ; \
 	${PREPARE_COMMAND} \
-	$(MAKE) git-init || exit 1 ; \
 	$(MAKE) update fmt || exit 1 ; \
 	git config user.email "beats-jenkins@test.com" || exit 1 ; \
 	git config user.name "beats-jenkins" || exit 1 ; \
-	$(MAKE) git-add || exit 1 ; \
 	$(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \
 	$(MAKE) || exit 1 ; \
 	$(MAKE) unit
@@ -29,17 +26,21 @@ prepare-test:: python-env
 	# Makes sure to use current version of beats for testing
 	rm -fr ${BUILD_DIR}/src/github.com/elastic/beats/
 	git clone -s ${PWD}/../../ ${BUILD_DIR}/src/github.com/elastic/beats/
-
 	rm -fr ${BEAT_PATH}
 	mkdir -p ${BEAT_PATH}
+	cd ${PWD}/${BUILD_DIR}/src/github.com/elastic/beats/ ; \
+	export MODULE=elastic ; \
+	export METRICSET=test ; \
+	export GO111MODULE=off; \
 	export GOPATH=${PWD}/build ; \
-	. ${PYTHON_ENV}/bin/activate && \
-	  python ${PWD}/build/src/github.com/elastic/beats/script/generate.py \
-	    --type=${BEAT_TYPE} \
-	    --project_name=Testbeat \
-	    --github_name=ruflin \
-	    --beat_path=beatpath/testbeat \
-	    --full_name="Nicolas Ruflin"
+	export NEWBEAT_PROJECT_NAME=Testbeat ; \
+	export NEWBEAT_GITHUB_NAME=ruflin ; \
+	export NEWBEAT_BEAT_PATH=beatpath/testbeat ; \
+	export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \
+	export NEWBEAT_TYPE=${BEAT_TYPE} ; \
+	export NEWBEAT_DEV=1 ; \
+	mage GenerateCustomBeat
+
 
 # Runs test build for the created beat
 .PHONY: test-build
diff --git a/generator/metricbeat/{beat}/main.go b/generator/metricbeat/{beat}/main.go
index 793c64f756b..223a026ff88 100644
--- a/generator/metricbeat/{beat}/main.go
+++ b/generator/metricbeat/{beat}/main.go
@@ -9,7 +9,6 @@ import (
 	_ "{beat_path}/include"
 )
 
-
 func main() {
 	if err := cmd.RootCmd.Execute(); err != nil {
 		os.Exit(1)

From 492047ecb8e0cb6e7f798f593dfa5015e88e35af Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 24 Sep 2019 09:14:15 -0700
Subject: [PATCH 13/34] still trying to get CI to work

---
 generator/common/Makefile | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/generator/common/Makefile b/generator/common/Makefile
index 3c3be69f805..4fcac1f21fe 100644
--- a/generator/common/Makefile
+++ b/generator/common/Makefile
@@ -9,12 +9,8 @@ PREPARE_COMMAND?=
 # Runs test build for mock beat
 .PHONY: test
 test: prepare-test
-	. ${PYTHON_ENV}/bin/activate; \
-	export GOPATH=${PWD}/build ; \
-	export PATH=$${GOPATH}/bin:${PATH}; \
-	cd ${BEAT_PATH} ; \
-	${PREPARE_COMMAND} \
-	$(MAKE) update fmt || exit 1 ; \
+	cd ${GOPATH}/src/beatpath/testbeat ; \
+	export PATH=$${GOPATH}/bin:$${PATH}; \
 	git config user.email "beats-jenkins@test.com" || exit 1 ; \
 	git config user.name "beats-jenkins" || exit 1 ; \
 	$(MAKE) check CHECK_HEADERS_DISABLED=y || exit 1 ; \
@@ -22,17 +18,16 @@ test: prepare-test
 	$(MAKE) unit
 
 .PHONY: prepare-test
-prepare-test:: python-env
+prepare-test:: #python-env
 	# Makes sure to use current version of beats for testing
-	rm -fr ${BUILD_DIR}/src/github.com/elastic/beats/
-	git clone -s ${PWD}/../../ ${BUILD_DIR}/src/github.com/elastic/beats/
+	#rm -fr ${BUILD_DIR}/src/github.com/elastic/beats/
+	#git clone -s ${PWD}/../../ ${BUILD_DIR}/src/github.com/elastic/beats/
 	rm -fr ${BEAT_PATH}
 	mkdir -p ${BEAT_PATH}
-	cd ${PWD}/${BUILD_DIR}/src/github.com/elastic/beats/ ; \
+	cd ${GOPATH}/src/github.com/elastic/beats/ ; \
 	export MODULE=elastic ; \
 	export METRICSET=test ; \
 	export GO111MODULE=off; \
-	export GOPATH=${PWD}/build ; \
 	export NEWBEAT_PROJECT_NAME=Testbeat ; \
 	export NEWBEAT_GITHUB_NAME=ruflin ; \
 	export NEWBEAT_BEAT_PATH=beatpath/testbeat ; \

From 92f8d1d660b4245481ff53cd8c3b01d721d56d7d Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 24 Sep 2019 09:48:51 -0700
Subject: [PATCH 14/34] add install for mage

---
 generator/common/Makefile | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/generator/common/Makefile b/generator/common/Makefile
index 4fcac1f21fe..73842dc1db7 100644
--- a/generator/common/Makefile
+++ b/generator/common/Makefile
@@ -18,7 +18,7 @@ test: prepare-test
 	$(MAKE) unit
 
 .PHONY: prepare-test
-prepare-test:: #python-env
+prepare-test:: mage-setup
 	# Makes sure to use current version of beats for testing
 	#rm -fr ${BUILD_DIR}/src/github.com/elastic/beats/
 	#git clone -s ${PWD}/../../ ${BUILD_DIR}/src/github.com/elastic/beats/
@@ -47,6 +47,13 @@ test-build: test
 	$(MAKE) deps ; \
 	$(MAKE) images
 
+.PHONY: mage-setup
+mage-setup: ${GOPATH}/bin/mage
+	go get -u -d github.com/magefile/mage
+	cd $${GOPATH}/src/github.com/magefile/mage;  \
+	go run bootstrap.go
+
+
 # Sets up the virtual python environment
 .PHONY: python-env
 python-env:

From 4dbd3413269befbab633d36dfd793f36402ae720 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 24 Sep 2019 10:12:03 -0700
Subject: [PATCH 15/34] I forgot how make targets work

---
 generator/common/Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/generator/common/Makefile b/generator/common/Makefile
index 73842dc1db7..6489eb17cb5 100644
--- a/generator/common/Makefile
+++ b/generator/common/Makefile
@@ -18,7 +18,7 @@ test: prepare-test
 	$(MAKE) unit
 
 .PHONY: prepare-test
-prepare-test:: mage-setup
+prepare-test:: ${GOPATH}/bin/mage
 	# Makes sure to use current version of beats for testing
 	#rm -fr ${BUILD_DIR}/src/github.com/elastic/beats/
 	#git clone -s ${PWD}/../../ ${BUILD_DIR}/src/github.com/elastic/beats/
@@ -47,8 +47,8 @@ test-build: test
 	$(MAKE) deps ; \
 	$(MAKE) images
 
-.PHONY: mage-setup
-mage-setup: ${GOPATH}/bin/mage
+
+${GOPATH}/bin/mage:
 	go get -u -d github.com/magefile/mage
 	cd $${GOPATH}/src/github.com/magefile/mage;  \
 	go run bootstrap.go

From 45f7fafe583765148a50a6f2671a4de4681482e3 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 24 Sep 2019 13:12:43 -0700
Subject: [PATCH 16/34] remove old python setup role

---
 generator/common/Makefile     | 7 -------
 generator/metricbeat/Makefile | 5 -----
 2 files changed, 12 deletions(-)

diff --git a/generator/common/Makefile b/generator/common/Makefile
index 6489eb17cb5..46e56967b1d 100644
--- a/generator/common/Makefile
+++ b/generator/common/Makefile
@@ -54,13 +54,6 @@ ${GOPATH}/bin/mage:
 	go run bootstrap.go
 
 
-# Sets up the virtual python environment
-.PHONY: python-env
-python-env:
-	@test -d ${PYTHON_ENV} || virtualenv ${PYTHON_ENV}
-	@${PYTHON_ENV}/bin/pip install --upgrade pip PyYAML
-	@# Work around pip bug. See: https://github.com/pypa/pip/issues/4464
-	@find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';'
 
 # Cleans up environment
 .PHONY: clean
diff --git a/generator/metricbeat/Makefile b/generator/metricbeat/Makefile
index 2441db7c351..27dd8e43e28 100644
--- a/generator/metricbeat/Makefile
+++ b/generator/metricbeat/Makefile
@@ -3,11 +3,6 @@ PREPARE_COMMAND = MODULE=elastic METRICSET=test make create-metricset;
 
 include ../common/Makefile
 
-.PHONY: prepare-test
-prepare-test:: python-env
-	mkdir -p ${BEAT_PATH}/scripts
-	rsync -a --exclude=build ${PWD}/../../metricbeat/scripts/generate_imports_helper.py ${BEAT_PATH}/scripts
-
 # Collects all dependencies and then calls update
 .PHONY: collect
 collect:

From 4aae95368e49028cda5ab6f804247c8df288a998 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 25 Sep 2019 10:24:30 -0700
Subject: [PATCH 17/34] clean up test makefiles

---
 generator/common/Makefile     | 18 +++++-------------
 generator/metricbeat/Makefile |  1 -
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/generator/common/Makefile b/generator/common/Makefile
index 46e56967b1d..a0740683606 100644
--- a/generator/common/Makefile
+++ b/generator/common/Makefile
@@ -1,15 +1,14 @@
-BUILD_DIR?=build
 PWD=$(shell pwd)
-PYTHON_ENV?=${BUILD_DIR}/python-env
 BEAT_TYPE?=beat
-BEAT_PATH=${BUILD_DIR}/src/beatpath/testbeat
+BEAT_NAME?=beatpath/test${BEAT_TYPE}
+BEAT_PATH=${GOPATH}/src/${BEAT_NAME}
 ES_BEATS=${GOPATH}/src/github.com/elastic/beats
 PREPARE_COMMAND?=
 
 # Runs test build for mock beat
 .PHONY: test
 test: prepare-test
-	cd ${GOPATH}/src/beatpath/testbeat ; \
+	cd ${BEAT_PATH} ; \
 	export PATH=$${GOPATH}/bin:$${PATH}; \
 	git config user.email "beats-jenkins@test.com" || exit 1 ; \
 	git config user.name "beats-jenkins" || exit 1 ; \
@@ -19,9 +18,6 @@ test: prepare-test
 
 .PHONY: prepare-test
 prepare-test:: ${GOPATH}/bin/mage
-	# Makes sure to use current version of beats for testing
-	#rm -fr ${BUILD_DIR}/src/github.com/elastic/beats/
-	#git clone -s ${PWD}/../../ ${BUILD_DIR}/src/github.com/elastic/beats/
 	rm -fr ${BEAT_PATH}
 	mkdir -p ${BEAT_PATH}
 	cd ${GOPATH}/src/github.com/elastic/beats/ ; \
@@ -30,13 +26,12 @@ prepare-test:: ${GOPATH}/bin/mage
 	export GO111MODULE=off; \
 	export NEWBEAT_PROJECT_NAME=Testbeat ; \
 	export NEWBEAT_GITHUB_NAME=ruflin ; \
-	export NEWBEAT_BEAT_PATH=beatpath/testbeat ; \
+	export NEWBEAT_BEAT_PATH=${BEAT_NAME} ; \
 	export NEWBEAT_FULL_NAME="Nicolas Ruflin" ; \
 	export NEWBEAT_TYPE=${BEAT_TYPE} ; \
 	export NEWBEAT_DEV=1 ; \
 	mage GenerateCustomBeat
 
-
 # Runs test build for the created beat
 .PHONY: test-build
 test-build: test
@@ -47,15 +42,12 @@ test-build: test
 	$(MAKE) deps ; \
 	$(MAKE) images
 
-
 ${GOPATH}/bin/mage:
 	go get -u -d github.com/magefile/mage
 	cd $${GOPATH}/src/github.com/magefile/mage;  \
 	go run bootstrap.go
 
-
-
 # Cleans up environment
 .PHONY: clean
 clean:
-	@rm -rf build
+	rm -rf ${BEAT_PATH}
diff --git a/generator/metricbeat/Makefile b/generator/metricbeat/Makefile
index 27dd8e43e28..c773d75926c 100644
--- a/generator/metricbeat/Makefile
+++ b/generator/metricbeat/Makefile
@@ -1,5 +1,4 @@
 BEAT_TYPE       = metricbeat
-PREPARE_COMMAND = MODULE=elastic METRICSET=test make create-metricset;
 
 include ../common/Makefile
 

From 083fde440ada847034093fcfcd067b35c7b84ff9 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 25 Sep 2019 10:22:46 -0700
Subject: [PATCH 18/34] remove old script

---
 script/generate.py | 123 ---------------------------------------------
 1 file changed, 123 deletions(-)
 delete mode 100644 script/generate.py

diff --git a/script/generate.py b/script/generate.py
deleted file mode 100644
index 12800cab551..00000000000
--- a/script/generate.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import argparse
-import datetime
-import os
-
-# Creates a new beat or metricbeat based on the given parameters
-
-project_name = ""
-github_name = ""
-beat = ""
-beat_path = ""
-full_name = ""
-
-
-def generate_beat(args):
-
-    global project_name, github_name, beat, beat_path, full_name
-
-    if args.project_name is not None:
-        project_name = args.project_name
-
-    if args.github_name is not None:
-        github_name = args.github_name
-
-    if args.beat_path is not None:
-        beat_path = args.beat_path
-
-    if args.full_name is not None:
-        full_name = args.full_name
-
-    read_input()
-    process_file(args.type)
-
-
-def read_input():
-    """Requests input form the command line for empty variables if needed.
-    """
-    global project_name, github_name, beat, beat_path, full_name
-
-    if project_name == "":
-        project_name = raw_input("Beat Name [Examplebeat]: ") or "examplebeat"
-
-    if github_name == "":
-        github_name = raw_input("Your Github Name [your-github-name]: ") or "your-github-name"
-    beat = project_name.lower()
-
-    if beat_path == "":
-        beat_path = raw_input("Beat Path [github.com/" + github_name + "/" + beat +
-                              "]: ") or "github.com/" + github_name + "/" + beat
-
-    if full_name == "":
-        full_name = raw_input("Firstname Lastname: ") or "Firstname Lastname"
-
-
-def process_file(beat_type):
-
-    # Load path information
-    template_path = os.path.dirname(os.path.realpath(__file__)) + '/../generator'
-    go_path = os.environ['GOPATH']
-
-    for root, dirs, files in os.walk(template_path + '/' + beat_type + '/{beat}'):
-
-        for file in files:
-
-            full_path = root + "/" + file
-
-            # load file
-            content = ""
-            with open(full_path) as f:
-                content = f.read()
-
-            # process content
-            content = replace_variables(content)
-
-            # Write new path
-            new_path = replace_variables(full_path).replace(".go.tmpl", ".go")
-
-            # remove generator info and beat name from path
-            file_path = new_path.replace(template_path + "/" + beat_type + "/" + beat, "")
-
-            # New file path to write file content to
-            write_file = go_path + "/src/" + beat_path + "/" + file_path
-
-            # Create parent directory if it does not exist yet
-            dir = os.path.dirname(write_file)
-            if not os.path.exists(dir):
-                os.makedirs(dir)
-
-            # Write file to new location
-            with open(write_file, 'w') as f:
-                f.write(content)
-
-
-def replace_variables(content):
-    """Replace all template variables with the actual values
-    """
-    return content.replace("{project_name}", project_name) \
-        .replace("{github_name}", github_name) \
-        .replace("{beat}", beat) \
-        .replace("{Beat}", beat.capitalize()) \
-        .replace("{beat_path}", beat_path) \
-        .replace("{full_name}", full_name) \
-        .replace("{year}", str(datetime.datetime.now().year))
-
-
-def get_parser():
-    """Creates parser to parse script params
-    """
-    parser = argparse.ArgumentParser(description="Creates a beat")
-    parser.add_argument("--project_name", help="Project name")
-    parser.add_argument("--github_name", help="Github name")
-    parser.add_argument("--beat_path", help="Beat path")
-    parser.add_argument("--full_name", help="Full name")
-    parser.add_argument("--type", help="Beat type", default="beat", choices=["beat", "metricbeat"])
-
-    return parser
-
-
-if __name__ == "__main__":
-
-    parser = get_parser()
-    args = parser.parse_args()
-
-    generate_beat(args)

From c0efb4ab0079a4980c5b6ed3ea36a3cd0d912a8c Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 25 Sep 2019 12:01:42 -0700
Subject: [PATCH 19/34] update devguide

---
 .../creating-beat-from-metricbeat.asciidoc    | 26 +++++--------------
 docs/devguide/newbeat.asciidoc                | 15 +++++------
 2 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/docs/devguide/creating-beat-from-metricbeat.asciidoc b/docs/devguide/creating-beat-from-metricbeat.asciidoc
index 77b55c48707..b357134ed0e 100644
--- a/docs/devguide/creating-beat-from-metricbeat.asciidoc
+++ b/docs/devguide/creating-beat-from-metricbeat.asciidoc
@@ -43,34 +43,20 @@ This directory is normally located under `$GOPATH/src/github.com/{your-github-na
 [float]
 ==== Step 2 - Create the Beat
 
-Run the command:
+Run the command from the root `beats` directory:
 
 [source,bash]
 ----
-python ${GOPATH}/src/github.com/elastic/beats/script/generate.py --type=metricbeat
+NEWBEAT_TYPE=metricbeat mage GenerateCustomBeat
 ----
 
-When prompted, enter the Beat name and path.
+When prompted, enter the Beat name and path, along with an initial module and metricset name for your beat. For more details about creating a metricset, see the docs <<creating-metricsets>>.
 
-
-[float]
-==== Step 3 - Init and create the metricset
-
-After creating the Beat, change the directory to `$GOPATH/src/github.com/{your-github-name}/{beat}` and run:
-
-[source,bash]
-----
-make setup
-----
-
-This will do the initial setup for your Beat and also run `make create-metricset`, which will ask you for the
-module name and metricset name of your Beat.
-
-For more details about creating a metricset, see the docs <<creating-metricsets>>.
+After creating the Beat, change the directory to `$GOPATH/src/github.com/{your-github-name}/{beat}`
 
 
 [float]
-==== Step 4 - Build & Run
+==== Step 3 - Build & Run
 
 To create a binary run the `make` command. This will create the binary in your beats directory.
 
@@ -84,7 +70,7 @@ To run it, execute the binary. This will automatically load the default configur
 This will run the beat with debug output enabled to the console to directly see what is happening. Stop the beat with `CTRL-C`.
 
 [float]
-==== Step 5 - Package
+==== Step 4 - Package
 
 To create packages and binaries for different platforms, https://www.docker.com/[docker] is required.
 The first step is to get the most recent packaging tools into your beat:
diff --git a/docs/devguide/newbeat.asciidoc b/docs/devguide/newbeat.asciidoc
index bc775a467d9..16971a37c01 100644
--- a/docs/devguide/newbeat.asciidoc
+++ b/docs/devguide/newbeat.asciidoc
@@ -122,24 +122,23 @@ mkdir ${GOPATH}/src/github.com/{user}
 cd ${GOPATH}/src/github.com/{user}
 --------------------
 
-Run python and specify the path to the Beat generator:
+Run the mage script to generate the custom beat:
 
-NOTE: Python 2 is required (Python 3 will not work).
 
 [source,shell]
 --------------------
-python $GOPATH/src/github.com/elastic/beats/script/generate.py
+mage GenerateCustomBeat
 --------------------
 
-Python will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`.
+The mage script will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`.
 For the `github_name`, enter your github id. The `beat` and `beat_path` are set to the correct values automatically (just press Enter to accept each default). For the `full_name`, enter your firstname and lastname.
 
 [source,shell]
 ---------
-Beat Name [Examplebeat]: Countbeat
-Your Github Name [your-github-name]: {username}
-Beat Path [github.com/{github id}/{beat name}]:
-Firstname Lastname: {Full Name}
+Enter a project name [examplebeat]: 
+Enter a github name [your-github-name]: 
+Enter a beat path [github.com/your-github-name/examplebeat]: 
+Enter a full name [Firstname Lastname]:
 ---------
 
 The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat).

From 5a64748211208a7cc870529abd6ccbb34f09f20f Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 25 Sep 2019 13:32:29 -0700
Subject: [PATCH 20/34] fix example in Newbeat guide

---
 docs/devguide/newbeat.asciidoc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/devguide/newbeat.asciidoc b/docs/devguide/newbeat.asciidoc
index 16971a37c01..d70a565e461 100644
--- a/docs/devguide/newbeat.asciidoc
+++ b/docs/devguide/newbeat.asciidoc
@@ -135,10 +135,10 @@ For the `github_name`, enter your github id. The `beat` and `beat_path` are set
 
 [source,shell]
 ---------
-Enter a project name [examplebeat]: 
-Enter a github name [your-github-name]: 
-Enter a beat path [github.com/your-github-name/examplebeat]: 
-Enter a full name [Firstname Lastname]:
+Enter a project name [examplebeat]:  Countbeat
+Enter a github name [your-github-name]: {username}
+Enter a beat path [github.com/{username}/countbeat]: 
+Enter a full name [Firstname Lastname]: {Full Name}
 ---------
 
 The Beat generator creates a directory called `countbeat` inside of your project folder (e.g. {project folder}/github.com/{github id}/countbeat).

From 8a3980700acb929217a30468e30d38b7a267bea9 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 25 Sep 2019 13:36:15 -0700
Subject: [PATCH 21/34] remove lic header in template magefile

---
 generator/beat/{beat}/magefile.go | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/generator/beat/{beat}/magefile.go b/generator/beat/{beat}/magefile.go
index af1c10043cd..1c587749c4d 100644
--- a/generator/beat/{beat}/magefile.go
+++ b/generator/beat/{beat}/magefile.go
@@ -1,20 +1,3 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
 // +build mage
 
 package main

From b6774b46488e3f396ecfff0f7615424fee38d30d Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 25 Sep 2019 13:40:17 -0700
Subject: [PATCH 22/34] code cleanup

---
 generator/common/beatgen/beatgen.go     | 2 --
 generator/common/beatgen/setup/setup.go | 5 -----
 2 files changed, 7 deletions(-)

diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index 501f8ccc180..cc131ab649b 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -113,7 +113,6 @@ func Generate() error {
 
 // VendorUpdate updates the beat vendor directory
 func VendorUpdate() error {
-
 	err := sh.Rm("./vendor/github.com/elastic/beats")
 	if err != nil {
 		return errors.Wrap(err, "error removing vendor dir")
@@ -174,7 +173,6 @@ func getValFromUser(cfgKey, def string) (string, error) {
 
 // getAbsoluteBeatsPath tries to infer the "real" non-vendor beats path
 func getAbsoluteBeatsPath() string {
-
 	beatsImportPath := "github.com/elastic/beats"
 	gopath := os.Getenv("GOPATH")
 	if gopath != "" {
diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go
index 50504e84593..82014c813cb 100644
--- a/generator/common/beatgen/setup/setup.go
+++ b/generator/common/beatgen/setup/setup.go
@@ -106,11 +106,6 @@ func CopyVendor() error {
 
 }
 
-// getRealBeatsRepo gets a non
-func getRealBeatsRepo() {
-
-}
-
 // GitInit initializes a new git repo in the current directory
 func GitInit() error {
 	return sh.Run("git", "init")

From b82d74f86b7c727d1160a9706866b3267d186d93 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 9 Oct 2019 13:32:25 -0700
Subject: [PATCH 23/34] error and doc cleanup

---
 docs/devguide/creating-beat-from-metricbeat.asciidoc | 1 +
 generator/common/beatgen/beatgen.go                  | 6 +++---
 generator/common/beatgen/setup/creator.go            | 6 +++---
 generator/common/beatgen/setup/setup.go              | 9 +++++----
 4 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/docs/devguide/creating-beat-from-metricbeat.asciidoc b/docs/devguide/creating-beat-from-metricbeat.asciidoc
index b357134ed0e..0e4c46736f9 100644
--- a/docs/devguide/creating-beat-from-metricbeat.asciidoc
+++ b/docs/devguide/creating-beat-from-metricbeat.asciidoc
@@ -12,6 +12,7 @@ must be set up correctly. In addition, the following tools are required:
 
 * https://www.python.org/downloads/[python]
 * https://virtualenv.pypa.io/en/stable/[virtualenv]
+* https://github.com/magefile/mage[mage]
 
 Virtualenv is easiest installed with your package manager or https://pip.pypa.io/en/stable/[pip].
 
diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index cc131ab649b..c04704c522f 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -79,18 +79,18 @@ var configList = []ConfigItem{
 func Generate() error {
 	cfg, err := getConfig()
 	if err != nil {
-		return err
+		return errors.Wrap(err, "Error getting config")
 	}
 	err = setup.GenNewBeat(cfg)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "Error generating new beat")
 	}
 
 	absBeatPath := filepath.Join(build.Default.GOPATH, "src", cfg["beat_path"])
 
 	err = os.Chdir(absBeatPath)
 	if err != nil {
-		return err
+		return errors.Wrap(err, "error changing directory")
 	}
 
 	mg.Deps(setup.CopyVendor)
diff --git a/generator/common/beatgen/setup/creator.go b/generator/common/beatgen/setup/creator.go
index e10b9d17237..87c95b844aa 100644
--- a/generator/common/beatgen/setup/creator.go
+++ b/generator/common/beatgen/setup/creator.go
@@ -49,20 +49,20 @@ func GenNewBeat(config map[string]string) error {
 		if info.IsDir() {
 			err := os.MkdirAll(writePath, 0755)
 			if err != nil {
-				return errors.Wrap(err, "error creating directory")
+				return errors.Wrapf(err, "error creating directory %s", writePath)
 			}
 		} else {
 
 			//dump original source file
 			tmplFile, err := ioutil.ReadFile(path)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "error reading source templatse file")
 			}
 			newFile := replaceVars(config, string(tmplFile))
 
 			err = ioutil.WriteFile(writePath, []byte(newFile), 0644)
 			if err != nil {
-				return err
+				return errors.Wrap(err, "error writing beat file")
 			}
 		}
 
diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go
index 82014c813cb..0c2cf763a7e 100644
--- a/generator/common/beatgen/setup/setup.go
+++ b/generator/common/beatgen/setup/setup.go
@@ -32,20 +32,21 @@ func RunSetup() error {
 	vendorPath := "./vendor/github.com/"
 
 	//Copy mage stuff
-	err := os.MkdirAll(filepath.Join(vendorPath, "magefile"), 0755)
+	toMkdir := filepath.Join(vendorPath, "magefile")
+	err := os.MkdirAll(toMkdir, 0755)
 	if err != nil {
-		return errors.Wrap(err, "error making mage dir")
+		return errors.Wrapf(err, "error making mage directory at %s", toMkdir)
 	}
 
 	err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/magefile/mage"), filepath.Join(vendorPath, "magefile"))
 	if err != nil {
-		return errors.Wrap(err, "error copying vendored magefile")
+		return errors.Wrapf(err, "error copying vendored magefile to %s", filepath.Join(vendorPath, "magefile"))
 	}
 
 	//Copy the pkg helper
 	err = sh.Run("cp", "-R", filepath.Join(vendorPath, "elastic/beats/vendor/github.com/pkg"), vendorPath)
 	if err != nil {
-		return errors.Wrap(err, "error copying pkg to vendor")
+		return errors.Wrapf(err, "error copying pkg to %s", vendorPath)
 	}
 
 	pwd, err := os.Getwd()

From de7fd1f5c9d33b041dbcbfb998416f854af16a87 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 9 Oct 2019 13:45:14 -0700
Subject: [PATCH 24/34] run the imports helper link when we update vendor

---
 generator/common/beatgen/beatgen.go     | 2 +-
 generator/common/beatgen/setup/setup.go | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index c04704c522f..a9ffaffc945 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -119,7 +119,7 @@ func VendorUpdate() error {
 	}
 
 	devtools.SetElasticBeatsDir(getAbsoluteBeatsPath())
-	mg.Deps(setup.CopyVendor)
+	mg.SerialDeps(setup.CopyVendor, setup.LinkImportsHelper)
 	return nil
 }
 
diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go
index 0c2cf763a7e..9db59bd6255 100644
--- a/generator/common/beatgen/setup/setup.go
+++ b/generator/common/beatgen/setup/setup.go
@@ -48,14 +48,17 @@ func RunSetup() error {
 	if err != nil {
 		return errors.Wrapf(err, "error copying pkg to %s", vendorPath)
 	}
+	return LinkImportsHelper()
+}
 
+// LinkImportsHelper links generate_imports_helper.py
+func LinkImportsHelper() error {
+	vendorPath := "./vendor/github.com/"
 	pwd, err := os.Getwd()
 	if err != nil {
 		return errors.Wrap(err, "error gettting current directory")
 	}
-
 	return sh.Run("ln", "-sf", filepath.Join(pwd, vendorPath, "elastic/beats/metricbeat/scripts/generate_imports_helper.py"), filepath.Join(pwd, vendorPath, "elastic/beats/script/generate_imports_helper.py"))
-
 }
 
 // CopyVendor copies a new version of beats into the vendor directory of PWD

From 48bcaaecfbf2a8fa554d1f851580866894121470 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Thu, 10 Oct 2019 08:27:37 -0700
Subject: [PATCH 25/34] print message with newbeat path

---
 generator/common/beatgen/beatgen.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index a9ffaffc945..953976a21f0 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -108,6 +108,10 @@ func Generate() error {
 	mg.Deps(setup.Update)
 	mg.Deps(setup.GitAdd)
 
+	fmt.Printf("=======================\n")
+	fmt.Printf("Your custom beat is now available as %s\n", absBeatPath)
+	fmt.Printf("=======================\n")
+
 	return nil
 }
 

From 5784861bb7cce459a20591ea2907ebff61e0c746 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Fri, 11 Oct 2019 07:15:59 -0700
Subject: [PATCH 26/34] restore old script, move prefix to const

---
 generator/common/beatgen/setup/creator.go |   2 +-
 script/generate.py                        | 123 ++++++++++++++++++++++
 2 files changed, 124 insertions(+), 1 deletion(-)
 create mode 100644 script/generate.py

diff --git a/generator/common/beatgen/setup/creator.go b/generator/common/beatgen/setup/creator.go
index 87c95b844aa..26859f101bb 100644
--- a/generator/common/beatgen/setup/creator.go
+++ b/generator/common/beatgen/setup/creator.go
@@ -30,7 +30,7 @@ import (
 )
 
 // CfgPrefix specifies the env variable prefix used to configure the beat
-var CfgPrefix = "NEWBEAT"
+const CfgPrefix = "NEWBEAT"
 
 // GenNewBeat generates a new custom beat
 // We assume our config object is populated and valid here
diff --git a/script/generate.py b/script/generate.py
new file mode 100644
index 00000000000..12800cab551
--- /dev/null
+++ b/script/generate.py
@@ -0,0 +1,123 @@
+import argparse
+import datetime
+import os
+
+# Creates a new beat or metricbeat based on the given parameters
+
+project_name = ""
+github_name = ""
+beat = ""
+beat_path = ""
+full_name = ""
+
+
+def generate_beat(args):
+
+    global project_name, github_name, beat, beat_path, full_name
+
+    if args.project_name is not None:
+        project_name = args.project_name
+
+    if args.github_name is not None:
+        github_name = args.github_name
+
+    if args.beat_path is not None:
+        beat_path = args.beat_path
+
+    if args.full_name is not None:
+        full_name = args.full_name
+
+    read_input()
+    process_file(args.type)
+
+
+def read_input():
+    """Requests input form the command line for empty variables if needed.
+    """
+    global project_name, github_name, beat, beat_path, full_name
+
+    if project_name == "":
+        project_name = raw_input("Beat Name [Examplebeat]: ") or "examplebeat"
+
+    if github_name == "":
+        github_name = raw_input("Your Github Name [your-github-name]: ") or "your-github-name"
+    beat = project_name.lower()
+
+    if beat_path == "":
+        beat_path = raw_input("Beat Path [github.com/" + github_name + "/" + beat +
+                              "]: ") or "github.com/" + github_name + "/" + beat
+
+    if full_name == "":
+        full_name = raw_input("Firstname Lastname: ") or "Firstname Lastname"
+
+
+def process_file(beat_type):
+
+    # Load path information
+    template_path = os.path.dirname(os.path.realpath(__file__)) + '/../generator'
+    go_path = os.environ['GOPATH']
+
+    for root, dirs, files in os.walk(template_path + '/' + beat_type + '/{beat}'):
+
+        for file in files:
+
+            full_path = root + "/" + file
+
+            # load file
+            content = ""
+            with open(full_path) as f:
+                content = f.read()
+
+            # process content
+            content = replace_variables(content)
+
+            # Write new path
+            new_path = replace_variables(full_path).replace(".go.tmpl", ".go")
+
+            # remove generator info and beat name from path
+            file_path = new_path.replace(template_path + "/" + beat_type + "/" + beat, "")
+
+            # New file path to write file content to
+            write_file = go_path + "/src/" + beat_path + "/" + file_path
+
+            # Create parent directory if it does not exist yet
+            dir = os.path.dirname(write_file)
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+
+            # Write file to new location
+            with open(write_file, 'w') as f:
+                f.write(content)
+
+
+def replace_variables(content):
+    """Replace all template variables with the actual values
+    """
+    return content.replace("{project_name}", project_name) \
+        .replace("{github_name}", github_name) \
+        .replace("{beat}", beat) \
+        .replace("{Beat}", beat.capitalize()) \
+        .replace("{beat_path}", beat_path) \
+        .replace("{full_name}", full_name) \
+        .replace("{year}", str(datetime.datetime.now().year))
+
+
+def get_parser():
+    """Creates parser to parse script params
+    """
+    parser = argparse.ArgumentParser(description="Creates a beat")
+    parser.add_argument("--project_name", help="Project name")
+    parser.add_argument("--github_name", help="Github name")
+    parser.add_argument("--beat_path", help="Beat path")
+    parser.add_argument("--full_name", help="Full name")
+    parser.add_argument("--type", help="Beat type", default="beat", choices=["beat", "metricbeat"])
+
+    return parser
+
+
+if __name__ == "__main__":
+
+    parser = get_parser()
+    args = parser.parse_args()
+
+    generate_beat(args)

From e29c38bb3364691e37a27b34dd04085231e4a8c6 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Mon, 14 Oct 2019 10:04:23 -0700
Subject: [PATCH 27/34] change logic that decides how we copy vendor

---
 generator/common/beatgen/setup/setup.go | 28 +++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/generator/common/beatgen/setup/setup.go b/generator/common/beatgen/setup/setup.go
index 9db59bd6255..7bebb0a735a 100644
--- a/generator/common/beatgen/setup/setup.go
+++ b/generator/common/beatgen/setup/setup.go
@@ -74,9 +74,15 @@ func CopyVendor() error {
 	if err != nil {
 		return errors.Wrap(err, "error creating vendor dir")
 	}
-	if _, isDev := os.LookupEnv(CfgPrefix + "_DEV"); isDev {
+
+	isClean, err := checkBeatsDirClean()
+	if err != nil {
+		return errors.Wrap(err, "error in checkIfBeatsDirClean")
+	}
+
+	if !isClean {
 		//Dev mode. Use CP.
-		fmt.Printf("CopyVendor running in dev mode, elastic/beats will be copied into the vendor directory with cp\n")
+		fmt.Printf("You have uncommited changes in elastic/beats. Running CopyVendor running in dev mode, elastic/beats will be copied into the vendor directory with cp\n")
 		vendorPath = filepath.Join(vendorPath, "beats")
 
 		err = sh.Run("cp", "-R", beatPath, vendorPath)
@@ -110,6 +116,24 @@ func CopyVendor() error {
 
 }
 
+// checkIfBeatsDirClean checks to see if the working elastic/beats dir is modified.
+// If it is, we'll use a different method to copy beats to vendor/
+func checkBeatsDirClean() (bool, error) {
+	beatPath, err := devtools.ElasticBeatsDir()
+	if err != nil {
+		return false, errors.Wrap(err, "Could not find ElasticBeatsDir")
+	}
+	out, err := sh.Output("git", "-C", beatPath, "status", "--porcelain")
+	if err != nil {
+		return false, errors.Wrap(err, "Error checking status of elastic/beats repo")
+	}
+
+	if len(out) == 0 {
+		return true, nil
+	}
+	return false, nil
+}
+
 // GitInit initializes a new git repo in the current directory
 func GitInit() error {
 	return sh.Run("git", "init")

From e41a52314841234acf04fb81fc4db8774a2c2c26 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Mon, 14 Oct 2019 10:15:30 -0700
Subject: [PATCH 28/34] add deprecation message to old python script

---
 script/generate.py | 124 +--------------------------------------------
 1 file changed, 2 insertions(+), 122 deletions(-)

diff --git a/script/generate.py b/script/generate.py
index 12800cab551..4ce4e892567 100644
--- a/script/generate.py
+++ b/script/generate.py
@@ -1,123 +1,3 @@
-import argparse
-import datetime
-import os
-
-# Creates a new beat or metricbeat based on the given parameters
-
-project_name = ""
-github_name = ""
-beat = ""
-beat_path = ""
-full_name = ""
-
-
-def generate_beat(args):
-
-    global project_name, github_name, beat, beat_path, full_name
-
-    if args.project_name is not None:
-        project_name = args.project_name
-
-    if args.github_name is not None:
-        github_name = args.github_name
-
-    if args.beat_path is not None:
-        beat_path = args.beat_path
-
-    if args.full_name is not None:
-        full_name = args.full_name
-
-    read_input()
-    process_file(args.type)
-
-
-def read_input():
-    """Requests input form the command line for empty variables if needed.
-    """
-    global project_name, github_name, beat, beat_path, full_name
-
-    if project_name == "":
-        project_name = raw_input("Beat Name [Examplebeat]: ") or "examplebeat"
-
-    if github_name == "":
-        github_name = raw_input("Your Github Name [your-github-name]: ") or "your-github-name"
-    beat = project_name.lower()
-
-    if beat_path == "":
-        beat_path = raw_input("Beat Path [github.com/" + github_name + "/" + beat +
-                              "]: ") or "github.com/" + github_name + "/" + beat
-
-    if full_name == "":
-        full_name = raw_input("Firstname Lastname: ") or "Firstname Lastname"
-
-
-def process_file(beat_type):
-
-    # Load path information
-    template_path = os.path.dirname(os.path.realpath(__file__)) + '/../generator'
-    go_path = os.environ['GOPATH']
-
-    for root, dirs, files in os.walk(template_path + '/' + beat_type + '/{beat}'):
-
-        for file in files:
-
-            full_path = root + "/" + file
-
-            # load file
-            content = ""
-            with open(full_path) as f:
-                content = f.read()
-
-            # process content
-            content = replace_variables(content)
-
-            # Write new path
-            new_path = replace_variables(full_path).replace(".go.tmpl", ".go")
-
-            # remove generator info and beat name from path
-            file_path = new_path.replace(template_path + "/" + beat_type + "/" + beat, "")
-
-            # New file path to write file content to
-            write_file = go_path + "/src/" + beat_path + "/" + file_path
-
-            # Create parent directory if it does not exist yet
-            dir = os.path.dirname(write_file)
-            if not os.path.exists(dir):
-                os.makedirs(dir)
-
-            # Write file to new location
-            with open(write_file, 'w') as f:
-                f.write(content)
-
-
-def replace_variables(content):
-    """Replace all template variables with the actual values
-    """
-    return content.replace("{project_name}", project_name) \
-        .replace("{github_name}", github_name) \
-        .replace("{beat}", beat) \
-        .replace("{Beat}", beat.capitalize()) \
-        .replace("{beat_path}", beat_path) \
-        .replace("{full_name}", full_name) \
-        .replace("{year}", str(datetime.datetime.now().year))
-
-
-def get_parser():
-    """Creates parser to parse script params
-    """
-    parser = argparse.ArgumentParser(description="Creates a beat")
-    parser.add_argument("--project_name", help="Project name")
-    parser.add_argument("--github_name", help="Github name")
-    parser.add_argument("--beat_path", help="Beat path")
-    parser.add_argument("--full_name", help="Full name")
-    parser.add_argument("--type", help="Beat type", default="beat", choices=["beat", "metricbeat"])
-
-    return parser
-
-
 if __name__ == "__main__":
-
-    parser = get_parser()
-    args = parser.parse_args()
-
-    generate_beat(args)
+    print "This script is deprecated. Please use `mage GenerateCustomBeat`"
+    exit(1)
\ No newline at end of file

From a77cd5bc2503a744953131cccc594dc54f556cdf Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Mon, 14 Oct 2019 10:52:00 -0700
Subject: [PATCH 29/34] remove unused make targets

---
 generator/beat/{beat}/Makefile       | 30 ----------------------------
 generator/metricbeat/{beat}/Makefile | 30 ----------------------------
 2 files changed, 60 deletions(-)

diff --git a/generator/beat/{beat}/Makefile b/generator/beat/{beat}/Makefile
index 3a96d91280d..247e315a7cb 100644
--- a/generator/beat/{beat}/Makefile
+++ b/generator/beat/{beat}/Makefile
@@ -13,33 +13,3 @@ CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(LIBBEAT_MAKEFILE)
-
-# Initial beat setup
-.PHONY: setup
-setup: pre-setup git-add
-
-pre-setup: copy-vendor git-init
-	$(MAKE) -f $(LIBBEAT_MAKEFILE) mage ES_BEATS=$(ES_BEATS)
-	$(MAKE) -f $(LIBBEAT_MAKEFILE) update BEAT_NAME=$(BEAT_NAME) ES_BEATS=$(ES_BEATS) NO_COLLECT=$(NO_COLLECT)
-
-# Copy beats into vendor directory
-.PHONY: copy-vendor
-copy-vendor: vendor-check
-	mkdir -p vendor/github.com/elastic/beats
-	git archive --remote ${BEAT_GOPATH}/src/github.com/elastic/beats HEAD | tar -x --exclude=x-pack -C vendor/github.com/elastic/beats
-	mkdir -p vendor/github.com/magefile
-	cp -R vendor/github.com/elastic/beats/vendor/github.com/magefile/mage vendor/github.com/magefile
-
-.PHONY: git-init
-git-init:
-	git init
-
-.PHONY: git-add
-git-add:
-	git add -A
-	git commit -q -m "Add generated {beat} files"
-
-
-.PHONY: vendor-check
-vendor-check:
-	@if output=$$(git -C ${BEAT_GOPATH}/src/github.com/elastic/beats status --porcelain) && [ ! -z "$${output}" ]; then printf "\033[31mWARNING: elastic/beats has uncommitted changes, these will not be in the vendor directory!\033[0m\n"; fi
\ No newline at end of file
diff --git a/generator/metricbeat/{beat}/Makefile b/generator/metricbeat/{beat}/Makefile
index 996964eab9e..9de701299d5 100644
--- a/generator/metricbeat/{beat}/Makefile
+++ b/generator/metricbeat/{beat}/Makefile
@@ -11,33 +11,3 @@ CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/metricbeat/Makefile
-
-# Initial beat setup
-.PHONY: setup
-setup: copy-vendor git-init
-	#call make recursively so we can reload the above include.
-	#Only needed during the first setup phase, before /vendor exists
-	$(MAKE) create-metricset update git-add
-
-# Copy beats into vendor directory
-.PHONY: copy-vendor
-copy-vendor: vendor-check
-	mkdir -p vendor/github.com/elastic/beats
-	git archive --remote ${BEAT_GOPATH}/src/github.com/elastic/beats HEAD | tar -x --exclude=x-pack -C vendor/github.com/elastic/beats
-	ln -sf ${PWD}/vendor/github.com/elastic/beats/metricbeat/scripts/generate_imports_helper.py ${PWD}/vendor/github.com/elastic/beats/script/generate_imports_helper.py
-	mkdir -p vendor/github.com/magefile
-	cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats/vendor/github.com/magefile/mage vendor/github.com/magefile
-	cp -R ${BEAT_GOPATH}/src/github.com/elastic/beats/vendor/github.com/pkg vendor/github.com/
-
-.PHONY: git-init
-git-init:
-	git init
-
-.PHONY: git-add
-git-add:
-	git add -A
-	git commit -q -m "Add generated {beat} files"
-
-.PHONY: vendor-check
-vendor-check:
-	@if output=$$(git -C ${BEAT_GOPATH}/src/github.com/elastic/beats status --porcelain) && [ ! -z "$${output}" ]; then printf "\033[31mWARNING: elastic/beats has uncommitted changes, these will not be in the vendor directory!\033[0m\n"; fi
\ No newline at end of file

From f0d1e7bd280fa254bc6f87703e0d7297111b67d9 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Mon, 14 Oct 2019 13:23:45 -0700
Subject: [PATCH 30/34] add newline to python script

---
 script/generate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/script/generate.py b/script/generate.py
index 4ce4e892567..21ea9f9f781 100644
--- a/script/generate.py
+++ b/script/generate.py
@@ -1,3 +1,3 @@
 if __name__ == "__main__":
     print "This script is deprecated. Please use `mage GenerateCustomBeat`"
-    exit(1)
\ No newline at end of file
+    exit(1)

From fe7098a5e8b2a3653ab450af14492f222d12857b Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 15 Oct 2019 07:19:59 -0700
Subject: [PATCH 31/34] change verbage for users

---
 generator/common/beatgen/beatgen.go | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/generator/common/beatgen/beatgen.go b/generator/common/beatgen/beatgen.go
index 953976a21f0..cdb3bd1e5c9 100644
--- a/generator/common/beatgen/beatgen.go
+++ b/generator/common/beatgen/beatgen.go
@@ -36,25 +36,29 @@ import (
 type ConfigItem struct {
 	Key     string
 	Default func(map[string]string) string
+	Help    string
 }
 
 // required user config for a custom beat
 // These are specified in env variables with newbeat_*
 var configList = []ConfigItem{
 	{
-		Key: "project_name",
+		Key:  "project_name",
+		Help: "Enter the beat name",
 		Default: func(cfg map[string]string) string {
 			return "examplebeat"
 		},
 	},
 	{
-		Key: "github_name",
+		Key:  "github_name",
+		Help: "Enter your github name",
 		Default: func(cfg map[string]string) string {
 			return "your-github-name"
 		},
 	},
 	{
-		Key: "beat_path",
+		Key:  "beat_path",
+		Help: "Enter the beat path",
 		Default: func(cfg map[string]string) string {
 			ghName, _ := cfg["github_name"]
 			beatName, _ := cfg["project_name"]
@@ -62,13 +66,15 @@ var configList = []ConfigItem{
 		},
 	},
 	{
-		Key: "full_name",
+		Key:  "full_name",
+		Help: "Enter your full name",
 		Default: func(cfg map[string]string) string {
 			return "Firstname Lastname"
 		},
 	},
 	{
-		Key: "type",
+		Key:  "type",
+		Help: "Enter the beat type",
 		Default: func(cfg map[string]string) string {
 			return "beat"
 		},
@@ -135,7 +141,7 @@ func getConfig() (map[string]string, error) {
 		var err error
 		cfgKey, isSet := getEnvConfig(cfgVal.Key)
 		if !isSet {
-			cfgKey, err = getValFromUser(cfgVal.Key, cfgVal.Default(userCfg))
+			cfgKey, err = getValFromUser(cfgVal.Help, cfgVal.Default(userCfg))
 			if err != nil {
 				return userCfg, err
 			}
@@ -159,11 +165,11 @@ func getEnvConfig(cfgKey string) (string, bool) {
 }
 
 // getValFromUser returns a config object from the user. If they don't enter one, fallback to the default
-func getValFromUser(cfgKey, def string) (string, error) {
+func getValFromUser(help, def string) (string, error) {
 	reader := bufio.NewReader(os.Stdin)
 
 	// Human-readable prompt
-	fmt.Printf("Enter a %s [%s]: ", strings.Replace(cfgKey, "_", " ", -1), def)
+	fmt.Printf("%s [%s]: ", help, def)
 	str, err := reader.ReadString('\n')
 	if err != nil {
 		return "", err

From 6c60e056d966a81b3c64e795a6c40e15164c6518 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 15 Oct 2019 09:34:15 -0700
Subject: [PATCH 32/34] add legacy make roles to makefiles

---
 generator/beat/{beat}/Makefile       | 4 ++++
 generator/metricbeat/{beat}/Makefile | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/generator/beat/{beat}/Makefile b/generator/beat/{beat}/Makefile
index 247e315a7cb..0246f73dc98 100644
--- a/generator/beat/{beat}/Makefile
+++ b/generator/beat/{beat}/Makefile
@@ -13,3 +13,7 @@ CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(LIBBEAT_MAKEFILE)
+
+.PHONY: copy-vendor
+copy-vendor:
+	mage vendorUpdate
\ No newline at end of file
diff --git a/generator/metricbeat/{beat}/Makefile b/generator/metricbeat/{beat}/Makefile
index 9de701299d5..932b0508378 100644
--- a/generator/metricbeat/{beat}/Makefile
+++ b/generator/metricbeat/{beat}/Makefile
@@ -11,3 +11,7 @@ CHECK_HEADERS_DISABLED=true
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/metricbeat/Makefile
+
+.PHONY: copy-vendor
+copy-vendor:
+	mage vendorUpdate
\ No newline at end of file

From e0459700135e0ad46a47bb4ad4288babf0ef9b19 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Tue, 15 Oct 2019 10:07:22 -0700
Subject: [PATCH 33/34] add changelog entry

---
 CHANGELOG-developer.next.asciidoc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
index 2aeaf33358a..f43d29a4635 100644
--- a/CHANGELOG-developer.next.asciidoc
+++ b/CHANGELOG-developer.next.asciidoc
@@ -24,6 +24,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - For "metricbeat style" generated custom beats, the mage target `GoTestIntegration` has changed to `GoIntegTest` and `GoTestUnit` has changed to `GoUnitTest`. {pull}13341[13341]
 - Build docker and kubernetes features only on supported platforms. {pull}13509[13509]
 - Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509]
+- The custom beat generator now uses mage instead of python {pull}13610[13610]
 
 ==== Bugfixes
 

From c7410e9c1d1058a1d4842487dd7466bbe19ea589 Mon Sep 17 00:00:00 2001
From: Alex Kristiansen <alex.kristiansen@elastic.co>
Date: Wed, 16 Oct 2019 07:46:29 -0700
Subject: [PATCH 34/34] change changelog entry

---
 CHANGELOG-developer.next.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
index f43d29a4635..2d740169c10 100644
--- a/CHANGELOG-developer.next.asciidoc
+++ b/CHANGELOG-developer.next.asciidoc
@@ -24,7 +24,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - For "metricbeat style" generated custom beats, the mage target `GoTestIntegration` has changed to `GoIntegTest` and `GoTestUnit` has changed to `GoUnitTest`. {pull}13341[13341]
 - Build docker and kubernetes features only on supported platforms. {pull}13509[13509]
 - Need to register new processors to be used in the JS processor in their `init` functions. {pull}13509[13509]
-- The custom beat generator now uses mage instead of python {pull}13610[13610]
+- The custom beat generator now uses mage instead of python, `mage GenerateCustomBeat` can be used to create a new beat, and `mage vendorUpdate` to update the vendored libbeat in a custom beat. {pull}13610[13610]
 
 ==== Bugfixes