Replace cookiecutter to generate beats by internal script (#2996)

Currently cookiecutter is needed to generate a beat. This adds an additional dependency to generate a beat and does not give us a lot of control over which checks should be executed during execution. As our template is quite simple it can be done with basic string replacements. A simple python script is now done for the task which leads to less inputs needed by the user and allows us to do checks like GOPATH setup etc.

The script could potentially be done in Golang but the setup of python is needed anyways for development.

Further changes:

* Rename .go files to .go.tmpl so they are not checked for correct syntax
* Update documentation

NOTES:
* Changes for the metricbeat generator will follow in a second PR.
* Blog post about creating a beat should be updated to point to 5.1. See #3197

generate/beat.py

@@ -0,0 +1,107 @@
+import os
+import argparse
+
+# Creates a new beat based on the given parameters
+
+project_name = ""
+github_name = ""
+beat = ""
+beat_path = ""
+full_name = ""
+
+def generate_beat():
+    read_input()
+    process_file()
+
+def read_input():
+    """Requests input form the command line for empty variables if needed.
+    """
+    global project_name, github_name, beat, beat_path, full_name
+
+    if project_name == "":
+        project_name = raw_input("Beat Name [Examplebeat]: ") or "examplebeat"
+
+    if github_name == "":
+        github_name = raw_input("Your Github Name [your-github-name]: ") or "your-github-name"
+    beat = project_name.lower()
+
+    if beat_path == "":
+        beat_path = raw_input("Beat Path [github.com/" + github_name + "]: ") or "github.com/" + github_name
+
+    if full_name == "":
+        full_name = raw_input("Firstname Lastname: ") or "Firstname Lastname"
+
+def process_file():
+
+    # Load path information
+    generator_path = os.path.dirname(os.path.realpath(__file__))
+    go_path = os.environ['GOPATH']
+
+    for root, dirs, files in os.walk(generator_path + '/beat/{beat}'):
+
+        for file in files:
+
+            full_path = root + "/" + file
+
+            ## load file
+            content = ""
+            with open(full_path) as f:
+                content = f.read()
+
+            # process content
+            content = replace_variables(content)
+
+            # Write new path
+            new_path = replace_variables(full_path).replace(".go.tmpl", ".go")
+
+            # remove generator info from path
+            file_path = new_path.replace(generator_path + "/beat/", "")
+
+            # New file path to write file content to
+            write_file = go_path + "/src/" + beat_path + "/" + file_path
+
+            # Create parent directory if it does not exist yet
+            dir = os.path.dirname(write_file)
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+
+            # Write file to new location
+            with open(write_file, 'w') as f:
+                f.write(content)
+
+def replace_variables(content):
+    """Replace all template variables with the actual values
+    """
+    return content.replace("{project_name}", project_name) \
+        .replace("{github_name}", github_name) \
+        .replace("{beat}", beat) \
+        .replace("{Beat}", beat.capitalize()) \
+        .replace("{beat_path}", beat_path) \
+        .replace("{full_name}", full_name)
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description="Creates a beat")
+    parser.add_argument("--project_name", help="Project name")
+    parser.add_argument("--github_name", help="Github name")
+    parser.add_argument("--beat_path", help="Beat path")
+    parser.add_argument("--full_name", help="Full name")
+
+    args = parser.parse_args()
+
+
+    if args.project_name is not None:
+        project_name = args.project_name
+
+    if args.github_name is not None:
+        github_name = args.github_name
+
+    if args.beat_path is not None:
+        beat_path = args.beat_path
+
+    if args.full_name is not None:
+        full_name = args.full_name
+
+    generate_beat()
+

generate/beat/CHANGELOG.md

@@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file based on the
 - Make testing dependent on local version of beats
 
 ### Backward Compatibility Breaks
+- Remove dependency on cookiecutter
 
 ### Bugfixes
 

generate/beat/Makefile

@@ -8,17 +8,13 @@ ES_BEATS=${GOPATH}/src/github.com/elastic/beats
 .PHONY: test
 test: python-env
 
-	# Create copy of cookiecutter for building with defaults
-	mkdir -p build/src/beatpath
-	cp -r \{\{cookiecutter.beat\}\} build
-	cp tests/cookiecutter.json build/
-
-	mkdir -p ${BEAT_PATH}
-	. ${PYTHON_ENV}/bin/activate && cookiecutter --no-input -o build/src/beatpath -f  build ;
-
 	# Makes sure to use current version of beats for testing
 	mkdir -p ${BUILD_DIR}/src/github.com/elastic/beats/
-	rsync -a --exclude=generate ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/
+	rsync -a --exclude=build ${PWD}/../../* ${BUILD_DIR}/src/github.com/elastic/beats/
+
+	mkdir -p ${BEAT_PATH}
+	export GOPATH=${PWD}/build ; \
+	. ${PYTHON_ENV}/bin/activate && python ${PWD}/build/src/github.com/elastic/beats/generate/beat.py --project_name=Testbeat --github_name=ruflin --beat_path=beatpath --full_name="Nicolas Ruflin"
 
 	. ${PYTHON_ENV}/bin/activate; \
 	export GOPATH=${PWD}/build ; \
@@ -45,7 +41,7 @@ test-build: test
 .PHONY: python-env
 python-env:
 	test -d ${PYTHON_ENV} || virtualenv ${PYTHON_ENV}
-	. ${PYTHON_ENV}/bin/activate && pip install --upgrade pip cookiecutter PyYAML
+	. ${PYTHON_ENV}/bin/activate && pip install --upgrade pip PyYAML
 
 # Cleans up environment
 .PHONY: clean

generate/beat/{beat}/.gitignore

@@ -0,0 +1,7 @@
+/.idea
+/build
+
+.DS_Store
+/{beat}
+/{beat}.test
+*.pyc

generate/beat/{{cookiecutter.beat}}/.travis.yml → generate/beat/{beat}/.travis.yml

@@ -28,10 +28,10 @@ addons:
 
 before_install:
   # Redo the travis setup but with the elastic/libbeat path. This is needed so the package path is correct
-  - mkdir -p $HOME/gopath/src/github.com/{{cookiecutter.github_name|lower}}/{{cookiecutter.beat}}/
-  - rsync -az ${TRAVIS_BUILD_DIR}/ $HOME/gopath/src/github.com/{{cookiecutter.github_name|lower}}/{{cookiecutter.beat}}/
-  - export TRAVIS_BUILD_DIR=$HOME/gopath/src/github.com/{{cookiecutter.github_name|lower}}/{{cookiecutter.beat}}/
-  - cd $HOME/gopath/src/github.com/{{cookiecutter.github_name|lower}}/{{cookiecutter.beat}}/
+  - mkdir -p $HOME/gopath/src/github.com/{github_name}/{beat}/
+  - rsync -az ${TRAVIS_BUILD_DIR}/ $HOME/gopath/src/github.com/{github_name}/{beat}/
+  - export TRAVIS_BUILD_DIR=$HOME/gopath/src/github.com/{github_name}/{beat}/
+  - cd $HOME/gopath/src/github.com/{github_name}/{beat}/
 
 install:
   - true

generate/beat/{{cookiecutter.beat}}/LICENSE → generate/beat/{beat}/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2016 {{cookiecutter.full_name}}
+Copyright (c) 2016 {full_name}
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

generate/beat/{{cookiecutter.beat}}/Makefile → generate/beat/{beat}/Makefile

@@ -1,5 +1,5 @@
-BEATNAME={{cookiecutter.beat}}
-BEAT_DIR={{cookiecutter.beat_path}}/{{cookiecutter.beat}}
+BEATNAME={beat}
+BEAT_DIR={beat_path}/{beat}
 BEAT_GOPATH=$(firstword $(subst :, ,${GOPATH}))
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
@@ -33,7 +33,7 @@ git-init:
 	git commit -m "Add git settings"
 	git add .
 	git reset -- .travis.yml
-	git commit -m "Add {{cookiecutter.beat}}"
+	git commit -m "Add {beat}"
 	git add .travis.yml
 	git commit -m "Add Travis CI"
 

generate/beat/{{cookiecutter.beat}}/README.md → generate/beat/{beat}/README.md

@@ -1,18 +1,18 @@
-# {{cookiecutter.beat|capitalize}}
+# {Beat}
 
-Welcome to {{cookiecutter.beat|capitalize}}.
+Welcome to {Beat}.
 
 Ensure that this folder is at the following location:
-`${GOPATH}/{{cookiecutter.beat_path}}`
+`${GOPATH}/{beat_path}`
 
-## Getting Started with {{cookiecutter.beat|capitalize}}
+## Getting Started with {Beat}
 
 ### Requirements
 
 * [Golang](https://golang.org/dl/) 1.7
 
 ### Init Project
-To get running with {{cookiecutter.beat|capitalize}} and also install the
+To get running with {Beat} and also install the
 dependencies, run the following command:
 
 ```
@@ -21,19 +21,19 @@ make setup
 
 It will create a clean git history for each major step. Note that you can always rewrite the history if you wish before pushing your changes.
 
-To push {{cookiecutter.beat|capitalize}} in the git repository, run the following commands:
+To push {Beat} in the git repository, run the following commands:
 
 ```
-git remote set-url origin https://{{cookiecutter.beat_path}}/{{cookiecutter.beat}}
+git remote set-url origin https://{beat_path}/{beat}
 git push origin master
 ```
 
 For further development, check out the [beat developer guide](https://www.elastic.co/guide/en/beats/libbeat/current/new-beat.html).
 
 ### Build
 
-To build the binary for {{cookiecutter.beat|capitalize}} run the command below. This will generate a binary
-in the same directory with the name {{cookiecutter.beat}}.
+To build the binary for {Beat} run the command below. This will generate a binary
+in the same directory with the name {beat}.
 
 ```
 make
@@ -42,16 +42,16 @@ make
 
 ### Run
 
-To run {{cookiecutter.beat|capitalize}} with debugging output enabled, run:
+To run {Beat} with debugging output enabled, run:
 
 ```
-./{{cookiecutter.beat}} -c {{cookiecutter.beat}}.yml -e -d "*"
+./{beat} -c {beat}.yml -e -d "*"
 ```
 
 
 ### Test
 
-To test {{cookiecutter.beat|capitalize}}, run the following command:
+To test {Beat}, run the following command:
 
 ```
 make testsuite
@@ -71,7 +71,7 @@ The test coverage is reported in the folder `./build/coverage/`
 
 Each beat has a template for the mapping in elasticsearch and a documentation for the fields
 which is automatically generated based on `etc/fields.yml`.
-To generate etc/{{cookiecutter.beat}}.template.json and etc/{{cookiecutter.beat}}.asciidoc
+To generate etc/{beat}.template.json and etc/{beat}.asciidoc
 
 ```
 make update
@@ -80,7 +80,7 @@ make update
 
 ### Cleanup
 
-To clean  {{cookiecutter.beat|capitalize}} source code, run the following commands:
+To clean  {Beat} source code, run the following commands:
 
 ```
 make fmt
@@ -96,12 +96,12 @@ make clean
 
 ### Clone
 
-To clone {{cookiecutter.beat|capitalize}} from the git repository, run the following commands:
+To clone {Beat} from the git repository, run the following commands:
 
 ```
-mkdir -p ${GOPATH%%:*}/{{cookiecutter.beat_path}}
-cd ${GOPATH%%:*}/{{cookiecutter.beat_path}}
-git clone https://{{cookiecutter.beat_path}}/{{cookiecutter.beat}}
+mkdir -p ${GOPATH}/{beat_path}
+cd ${GOPATH}/{beat_path}
+git clone https://{beat_path}/{beat}
 ```
 
 

generate/beat/{beat}/_meta/beat.yml

@@ -0,0 +1,7 @@
+################### {Beat} Configuration Example #########################
+
+############################# {Beat} ######################################
+
+{beat}:
+  # Defines how often an event is sent to the output
+  period: 1s

generate/beat/{{cookiecutter.beat}}/_meta/fields.yml → generate/beat/{beat}/_meta/fields.yml

@@ -1,5 +1,5 @@
-- key: {{ cookiecutter.beat }}
-  title: {{ cookiecutter.beat }}
+- key: {beat}
+  title: {beat}
   description:
   fields:
     - name: counter

generate/beat/{{cookiecutter.beat}}/beater/{{cookiecutter.beat}}.go → generate/beat/{beat}/beater/{beat}.go.tmpl

@@ -9,10 +9,10 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/publisher"
 
-	"{{cookiecutter.beat_path}}/{{cookiecutter.beat}}/config"
+	"{beat_path}/{beat}/config"
 )
 
-type {{cookiecutter.beat|capitalize}} struct {
+type {Beat} struct {
 	done   chan struct{}
 	config config.Config
 	client publisher.Client
@@ -25,15 +25,15 @@ func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) {
 		return nil, fmt.Errorf("Error reading config file: %v", err)
 	}
 
-	bt := &{{cookiecutter.beat|capitalize}}{
-		done: make(chan struct{}),
+	bt := &{Beat}{
+		done:   make(chan struct{}),
 		config: config,
 	}
 	return bt, nil
 }
 
-func (bt *{{cookiecutter.beat|capitalize}}) Run(b *beat.Beat) error {
-	logp.Info("{{cookiecutter.beat}} is running! Hit CTRL-C to stop it.")
+func (bt *{Beat}) Run(b *beat.Beat) error {
+	logp.Info("{beat} is running! Hit CTRL-C to stop it.")
 
 	bt.client = b.Publisher.Connect()
 	ticker := time.NewTicker(bt.config.Period)
@@ -56,7 +56,7 @@ func (bt *{{cookiecutter.beat|capitalize}}) Run(b *beat.Beat) error {
 	}
 }
 
-func (bt *{{cookiecutter.beat|capitalize}}) Stop() {
+func (bt *{Beat}) Stop() {
 	bt.client.Close()
 	close(bt.done)
 }

generate/beat/{beat}/docs/index.asciidoc

@@ -0,0 +1,5 @@
+= {Beat} Docs
+
+Welcome to the {Beat} documentation.
+
+

generate/beat/{{cookiecutter.beat}}/main.go → generate/beat/{beat}/main.go.tmpl

@@ -5,11 +5,11 @@ import (
 
 	"github.com/elastic/beats/libbeat/beat"
 
-	"{{cookiecutter.beat_path}}/{{cookiecutter.beat}}/beater"
+	"{beat_path}/{beat}/beater"
 )
 
 func main() {
-	err := beat.Run("{{cookiecutter.beat}}", "", beater.New)
+	err := beat.Run("{beat}", "", beater.New)
 	if err != nil {
 		os.Exit(1)
 	}

generate/beat/{{cookiecutter.beat}}/main_test.go → generate/beat/{beat}/main_test.go.tmpl

@@ -1,6 +1,6 @@
 package main
 
-// This file is mandatory as otherwise the {{cookiecutter.beat}}.test binary is not generated correctly.
+// This file is mandatory as otherwise the {beat}.test binary is not generated correctly.
 
 import (
 	"flag"