From 14460257a0e97312ec52403c653aa79892e940cf Mon Sep 17 00:00:00 2001
From: Lee Hinman <57081003+leehinman@users.noreply.github.com>
Date: Fri, 7 Feb 2020 11:11:26 -0600
Subject: [PATCH] [Filebeat] move create-[module,fileset,fields] to mage
 (#15836) (#16140)

- move create-[module,fileset,fields] to mage
- make mage create commands available in x-pack/filebeat
- change Makefile to use mage for create commands

(cherry picked from commit 90f79f93c0fdf3e62070e2ed3a0aa469196939ba)
---
 CHANGELOG.next.asciidoc                       |  1 +
 filebeat/Makefile                             | 12 ++---
 filebeat/magefile.go                          |  2 +
 filebeat/scripts/generator/fileset/main.go    | 52 -------------------
 .../main.go => mage/generate/fields.go}       | 30 +++++------
 filebeat/scripts/mage/generate/fileset.go     | 50 ++++++++++++++++++
 .../main.go => mage/generate/module.go}       | 41 +++++++--------
 x-pack/filebeat/magefile.go                   |  2 +
 8 files changed, 92 insertions(+), 98 deletions(-)
 delete mode 100644 filebeat/scripts/generator/fileset/main.go
 rename filebeat/scripts/{generator/module/main.go => mage/generate/fields.go} (54%)
 create mode 100644 filebeat/scripts/mage/generate/fileset.go
 rename filebeat/scripts/{generator/fields/main.go => mage/generate/module.go} (52%)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 29206470406..84a8dc8a8a4 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -184,6 +184,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Set event.outcome field based on googlecloud audit log output. {pull}15731[15731]
 - Add dashboard for AWS vpcflow fileset. {pull}16007[16007]
 - Add ECS tls fields to zeek:smtp,rdp,ssl and aws:s3access,elb {issue}15757[15757] {pull}15935[15936]
+- move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836]
 
 *Heartbeat*
 
diff --git a/filebeat/Makefile b/filebeat/Makefile
index 6e37e664325..940077b04cd 100644
--- a/filebeat/Makefile
+++ b/filebeat/Makefile
@@ -14,15 +14,15 @@ update: mage
 
 # Creates a new module. Requires the params MODULE.
 .PHONY: create-module
-create-module:
-	@go run ${ES_BEATS}/filebeat/scripts/generator/module/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE)
+create-module: mage
+	mage generate:module
 
 # Creates a new fileset. Requires the params MODULE and FILESET.
 .PHONY: create-fileset
-create-fileset:
-	@go run ${ES_BEATS}/filebeat/scripts/generator/fileset/main.go --path=$(PWD) --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET)
+create-fileset: mage
+	mage generate:fileset
 
 # Creates a fields.yml based on a pipeline.json file. Requires the params MODULE and FILESET.
 .PHONY: create-fields
-create-fields:
-	@go run ${ES_BEATS}/filebeat/scripts/generator/fields/main.go --beats_path=$(BEAT_GOPATH)/src/$(BEAT_PATH) --module=$(MODULE) --fileset=$(FILESET)
+create-fields: mage
+	mage generate:fields
diff --git a/filebeat/magefile.go b/filebeat/magefile.go
index c65503c87e8..0331512e45b 100644
--- a/filebeat/magefile.go
+++ b/filebeat/magefile.go
@@ -31,6 +31,8 @@ import (
 
 	// mage:import
 	"github.com/elastic/beats/dev-tools/mage/target/common"
+	// mage:import generate
+	_ "github.com/elastic/beats/filebeat/scripts/mage/generate"
 )
 
 func init() {
diff --git a/filebeat/scripts/generator/fileset/main.go b/filebeat/scripts/generator/fileset/main.go
deleted file mode 100644
index 320ddbc55ef..00000000000
--- a/filebeat/scripts/generator/fileset/main.go
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-
-	"github.com/elastic/beats/filebeat/generator/fileset"
-)
-
-func main() {
-	module := flag.String("module", "", "Name of the module")
-	filesetName := flag.String("fileset", "", "Name of the fileset")
-	modulesPath := flag.String("path", ".", "Path to the generated fileset")
-	beatsPath := flag.String("beats_path", ".", "Path to elastic/beats")
-	flag.Parse()
-
-	if *module == "" {
-		fmt.Println("Missing parameter: module")
-		os.Exit(1)
-	}
-
-	if *filesetName == "" {
-		fmt.Println("Missing parameter: fileset")
-		os.Exit(1)
-	}
-
-	err := fileset.Generate(*module, *filesetName, *modulesPath, *beatsPath)
-	if err != nil {
-		fmt.Printf("Cannot generate fileset: %v\n", err)
-		os.Exit(3)
-	}
-
-	fmt.Println("New fileset was generated, please check that module.yml file have proper fileset dashboard settings. After setting up Grok pattern in pipeline.json, please generate fields.yml")
-}
diff --git a/filebeat/scripts/generator/module/main.go b/filebeat/scripts/mage/generate/fields.go
similarity index 54%
rename from filebeat/scripts/generator/module/main.go
rename to filebeat/scripts/mage/generate/fields.go
index 737650d918d..9bc8a8af40a 100644
--- a/filebeat/scripts/generator/module/main.go
+++ b/filebeat/scripts/mage/generate/fields.go
@@ -15,32 +15,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package main
+package generate
 
 import (
-	"flag"
 	"fmt"
 	"os"
 
-	"github.com/elastic/beats/filebeat/generator/module"
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	genfields "github.com/elastic/beats/filebeat/generator/fields"
 )
 
-func main() {
-	moduleName := flag.String("module", "", "Name of the module")
-	modulePath := flag.String("path", ".", "Path to the generated fileset")
-	beatsPath := flag.String("beats_path", ".", "Path to elastic/beats")
-	flag.Parse()
+// Fields creates a new fields.yml for an existing Filebeat fileset.
+// Use MODULE=module to specify the name of the existing module
+// Use FILESET=fileset to specify the name of the existing fileset
+func Fields() error {
+	targetModule := os.Getenv("MODULE")
+	targetFileset := os.Getenv("FILESET")
 
-	if *moduleName == "" {
-		fmt.Println("Missing parameter: module")
-		os.Exit(1)
+	if targetModule == "" || targetFileset == "" {
+		return fmt.Errorf("you must specify the module and fileset: MODULE=module FILESET=fileset mage generate:fields")
 	}
 
-	err := module.Generate(*moduleName, *modulePath, *beatsPath)
-	if err != nil {
-		fmt.Printf("Cannot generate module: %v\n", err)
-		os.Exit(2)
-	}
+	curDir := devtools.CWD()
 
-	fmt.Println("New module was generated, now you can start creating filesets by create-fileset command.")
+	return genfields.Generate(curDir, targetModule, targetFileset, false)
 }
diff --git a/filebeat/scripts/mage/generate/fileset.go b/filebeat/scripts/mage/generate/fileset.go
new file mode 100644
index 00000000000..612567a1104
--- /dev/null
+++ b/filebeat/scripts/mage/generate/fileset.go
@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package generate
+
+import (
+	"fmt"
+	"os"
+
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	genfileset "github.com/elastic/beats/filebeat/generator/fileset"
+)
+
+// Fileset creates a new fileset for an existing Filebeat module.
+// Use MODULE=module to specify the name of the existing module
+// Use FILESET=fileset to specify the name of the new fileset
+func Fileset() error {
+	targetModule := os.Getenv("MODULE")
+	targetFileset := os.Getenv("FILESET")
+
+	if targetModule == "" || targetFileset == "" {
+		return fmt.Errorf("you must specify the module and fileset: MODULE=module FILESET=fileset createFileset")
+	}
+
+	ossDir := devtools.OSSBeatDir()
+	xPackDir := devtools.XPackBeatDir()
+
+	switch devtools.CWD() {
+	case ossDir:
+		return genfileset.Generate(targetModule, targetFileset, ossDir, ossDir)
+	case xPackDir:
+		return genfileset.Generate(targetModule, targetFileset, xPackDir, ossDir)
+	default:
+		return fmt.Errorf("you must be in a filebeat directory")
+	}
+}
diff --git a/filebeat/scripts/generator/fields/main.go b/filebeat/scripts/mage/generate/module.go
similarity index 52%
rename from filebeat/scripts/generator/fields/main.go
rename to filebeat/scripts/mage/generate/module.go
index 523ebf82249..5d14e3ec09f 100644
--- a/filebeat/scripts/generator/fields/main.go
+++ b/filebeat/scripts/mage/generate/module.go
@@ -15,38 +15,33 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package main
+package generate
 
 import (
-	"flag"
 	"fmt"
 	"os"
 
-	"github.com/elastic/beats/filebeat/generator/fields"
+	devtools "github.com/elastic/beats/dev-tools/mage"
+	genmod "github.com/elastic/beats/filebeat/generator/module"
 )
 
-func main() {
-	module := flag.String("module", "", "Name of the module")
-	fileset := flag.String("fileset", "", "Name of the fileset")
-	beatsPath := flag.String("beats_path", ".", "Path to elastic/beats")
-	noDoc := flag.Bool("nodoc", false, "Generate documentation for fields")
-	flag.Parse()
-
-	if *module == "" {
-		fmt.Println("Missing parameter: module")
-		os.Exit(1)
+// Module creates a new Filebeat module.
+// Use MODULE=module to specify the name of the new module
+func Module() error {
+	targetModule := os.Getenv("MODULE")
+	if targetModule == "" {
+		return fmt.Errorf("you must specify the module: MODULE=name mage generate:module")
 	}
 
-	if *fileset == "" {
-		fmt.Println("Missing parameter: fileset")
-		os.Exit(1)
-	}
+	ossDir := devtools.OSSBeatDir()
+	xPackDir := devtools.XPackBeatDir()
 
-	err := fields.Generate(*beatsPath, *module, *fileset, *noDoc)
-	if err != nil {
-		fmt.Printf("Error while generating fields.yml: %v\n", err)
-		os.Exit(2)
+	switch devtools.CWD() {
+	case ossDir:
+		return genmod.Generate(targetModule, ossDir, ossDir)
+	case xPackDir:
+		return genmod.Generate(targetModule, xPackDir, ossDir)
+	default:
+		return fmt.Errorf("you must be in a filebeat directory")
 	}
-
-	fmt.Printf("Fields.yml generated for %s/%s\n", *module, *fileset)
 }
diff --git a/x-pack/filebeat/magefile.go b/x-pack/filebeat/magefile.go
index 4aa1d8851ce..d2d4f635765 100644
--- a/x-pack/filebeat/magefile.go
+++ b/x-pack/filebeat/magefile.go
@@ -18,6 +18,8 @@ import (
 
 	// mage:import
 	"github.com/elastic/beats/dev-tools/mage/target/common"
+	// mage:import generate
+	_ "github.com/elastic/beats/filebeat/scripts/mage/generate"
 )
 
 func init() {