Unify generator (#3452)

* Simplify beat creation

* Use one script create generic beat or metricbeat
* Rename `generate` directory to `template`
* Share most parts of the testing makefile
* Update docs to new logic
* Update travis build file

* one more renaming

.travis.yml

@@ -74,10 +74,10 @@ matrix:
 
     # Generators
     - os: linux
-      env: TARGETS="-C generate/metricbeat test"
+      env: TARGETS="-C generator/metricbeat test"
       go: *go_version
     - os: linux
-      env: TARGETS="-C generate/beat test"
+      env: TARGETS="-C generator/beat test"
       go: *go_version
 
 addons:

CHANGELOG.asciidoc

@@ -14,6 +14,8 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 
 *Affecting all Beats*
 
+- Change beat generator. Use `$GOPATH/src/github.com/elastic/beats/script/generate.py` to generate a beat. {pull}3452[3452]
+
 *Metricbeat*
 
 *Packetbeat*

Makefile

@@ -11,7 +11,7 @@ SNAPSHOT?=yes
 .PHONY: testsuite
 testsuite:
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;)
-	#$(MAKE) -C generate test
+	#$(MAKE) -C generator test
 
 stop-environments:
 	$(foreach var,$(PROJECTS_ENV),$(MAKE) -C $(var) stop-environment || exit 0;)
@@ -46,13 +46,13 @@ update:
 clean:
 	rm -rf build
 	$(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;)
-	$(MAKE) -C generate clean
+	$(MAKE) -C generator clean
 
 # Cleans up the vendor directory from unnecessary files
 # This should always be run after updating the dependencies
 .PHONY: clean-vendor
 clean-vendor:
-	sh scripts/clean_vendor.sh
+	sh script/clean_vendor.sh
 
 .PHONY: check
 check:
@@ -118,5 +118,3 @@ upload-release:
 .PHONY: notice
 notice:
 	python dev-tools/generate_notice.py .
-
-

generator/beat/Makefile

@@ -0,0 +1,3 @@
+
+
+include ../common/Makefile

generate/beat/Makefile → generator/common/Makefile

@@ -1,31 +1,35 @@
 BUILD_DIR?=build
 PWD=$(shell pwd)
-PYTHON_ENV?=${BUILD_DIR}/python-env/
+PYTHON_ENV?=${BUILD_DIR}/python-env
+BEAT_TYPE?=beat
 BEAT_PATH=${BUILD_DIR}/src/beatpath/testbeat
 ES_BEATS=${GOPATH}/src/github.com/elastic/beats
+PREPARE_COMMAND?=
 
 # Runs test build for mock beat
 .PHONY: test
-test: python-env
-
-	# Makes sure to use current version of beats for testing
-	mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/
-	rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/
-
-	mkdir -p ${BEAT_PATH}
-	export GOPATH=${PWD}/build ; \
-	. ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/generate/beat.py --project_name=Testbeat --github_name=ruflin --beat_path=beatpath/testbeat --full_name="Nicolas Ruflin"
+test: prepare-test
 
 	. ${PYTHON_ENV}/bin/activate; \
 	export GOPATH=${PWD}/build ; \
 	export PATH=${PATH}:${PWD}/build/bin; \
 	cd ${BEAT_PATH} ; \
 	make copy-vendor ; \
+	${PREPARE_COMMAND}  \
 	make check ; \
 	make update ; \
 	make ; \
 	make unit
 
+prepare-test: python-env
+	# Makes sure to use current version of beats for testing
+	mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/
+	rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/
+
+	mkdir -p ${BEAT_PATH}
+	export GOPATH=${PWD}/build ; \
+	. ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/script/generate.py --type=${BEAT_TYPE} --project_name=Testbeat --github_name=ruflin --beat_path=beatpath/testbeat --full_name="Nicolas Ruflin"
+
 # Runs test build for the created beat
 .PHONY: test-build
 test-build: test

generator/metricbeat/Makefile

@@ -0,0 +1,5 @@
+BEAT_TYPE=metricbeat
+PREPARE_COMMAND=MODULE=elastic METRICSET=test make create-metricset ;
+
+include ../common/Makefile
+

libbeat/docs/newbeat.asciidoc

@@ -142,7 +142,7 @@ Run python and specify the path to the Beat generator:
 
 [source,shell]
 --------------------
-python $GOPATH/src/github.com/elastic/beats/generate/beat.py
+python $GOPATH/src/github.com/elastic/beats/script/generate.py
 --------------------
 
 Python will prompt you to enter information about your Beat. For the `project_name`, enter `Countbeat`.

metricbeat/docs/developer-guide/creating-beat-from-metricbeat.asciidoc

@@ -47,7 +47,7 @@ Run the command:
 
 [source,bash]
 ----
-python ${GOPATH}/src/github.com/elastic/beats/generate/metricbeat.py
+python ${GOPATH}/src/github.com/elastic/beats/script/generate.py --type=metricbeat
 ----
 
 When prompted, enter the Beat name and path.

generate/helper.py → script/generate.py

@@ -1,7 +1,7 @@
 import os
 import argparse
 
-# Creates a new beat based on the given parameters
+# Creates a new beat or metricbeat based on the given parameters
 
 project_name = ""
 github_name = ""
@@ -10,7 +10,8 @@
 full_name = ""
 
 
-def generate_beat(template_path, args):
+def generate_beat(args):
+
 
     global project_name, github_name, beat, beat_path, full_name
 
@@ -27,7 +28,7 @@ def generate_beat(template_path, args):
         full_name = args.full_name
 
     read_input()
-    process_file(template_path)
+    process_file(args.type)
 
 
 def read_input():
@@ -50,13 +51,13 @@ def read_input():
         full_name = raw_input("Firstname Lastname: ") or "Firstname Lastname"
 
 
-def process_file(template_path):
+def process_file(beat_type):
 
     # Load path information
-    generator_path = os.path.dirname(os.path.realpath(__file__))
+    template_path = os.path.dirname(os.path.realpath(__file__)) + '/../generator'
     go_path = os.environ['GOPATH']
 
-    for root, dirs, files in os.walk(generator_path + '/' + template_path + '/{beat}'):
+    for root, dirs, files in os.walk(template_path + '/' + beat_type + '/{beat}'):
 
         for file in files:
 
@@ -74,7 +75,7 @@ def process_file(template_path):
             new_path = replace_variables(full_path).replace(".go.tmpl", ".go")
 
             # remove generator info and beat name from path
-            file_path = new_path.replace(generator_path + "/" + template_path + "/" + beat, "")
+            file_path = new_path.replace(template_path + "/" + beat_type + "/" + beat, "")
 
             # New file path to write file content to
             write_file = go_path + "/src/" + beat_path + "/" + file_path
@@ -108,5 +109,16 @@ def get_parser():
     parser.add_argument("--github_name", help="Github name")
     parser.add_argument("--beat_path", help="Beat path")
     parser.add_argument("--full_name", help="Full name")
+    parser.add_argument("--type", help="Beat type", default="beat")
 
     return parser
+
+
+if __name__ == "__main__":
+
+    parser = get_parser()
+    args = parser.parse_args()
+
+    generate_beat(args)
+
+