Skip to content

Commit

Permalink
Simplify template generation and handling
Browse files Browse the repository at this point in the history
* Only one file is passed instead of an array as now only file is needed. Introduce `-file` param.
* Local loading of beats version was removed as it can lean to cyclic imports
* Move `_meta/fields.full.generated.yml` to `fields.yml` in preparation for packaging `fields.yml` and ignore it in git
* Cleanup Template generation package
* Rename -beat.name param to -index as this describes it better

Part of elastic#3654
  • Loading branch information
ruflin committed Feb 27, 2017
1 parent 98dde8b commit e29ce83
Show file tree
Hide file tree
Showing 6 changed files with 65 additions and 47 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
/*/data
/*/logs
/*/_meta/kibana/index-pattern
/*/fields.yml

# Files
.DS_Store
Expand Down
35 changes: 18 additions & 17 deletions dev-tools/cmd/index_template/index_template.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,40 +13,41 @@ import (
// main generates index templates for the beats
func main() {

beatName := flag.String("beat.name", "", ": Base index name. Normally {beat_name} (required)")
beatVersion := beat.GetDefaultVersion()
index := flag.String("index", "", "Base index name. Normally {beat_name} (required)")
output := flag.String("output", "", "Required: Full path to the output file (required)")
version := flag.String("es.version", beat.GetDefaultVersion(), "Elasticsearch version")
version := flag.String("es.version", beatVersion, "Elasticsearch version")
file := flag.String("file", "", "Path to fields.yml file")

flag.Parse()

var existingFiles []string
files := flag.Args()

if len(files) == 0 {
fmt.Fprintf(os.Stderr, "No fields.yml files provided. At least one file must be added.")
if len(*file) == 0 {
fmt.Fprintf(os.Stderr, "File path cannot be empty")
os.Exit(1)
}

if *beatName == "" {
fmt.Fprintf(os.Stderr, "beat.name is empty. It must be set.")
if *index == "" {
fmt.Fprintf(os.Stderr, "index is empty. It must be set.")
os.Exit(1)
}

// Skip some of the passed files, as not all beats have the same files
for _, f := range files {
if _, err := os.Stat(f); err != nil {
fmt.Printf("Skipping file because it does not exist: %s", f)
continue
}
existingFiles = append(existingFiles, f)
if _, err := os.Stat(*file); err != nil {
fmt.Fprintf(os.Stderr, "Error during loading -file %s with error: %s", *file, err)
os.Exit(1)
}

// Make it compatible with the sem versioning
if *version == "2x" {
*version = "2.0.0"
}

templateString, err := template.GetTemplate(*version, *beatName, existingFiles)
tmpl, err := template.New(beatVersion, *version, *index)
if err != nil {
fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
os.Exit(1)
}

templateString, err := tmpl.Load(*file)
if err != nil {
fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
os.Exit(1)
Expand Down
8 changes: 4 additions & 4 deletions libbeat/scripts/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -247,17 +247,17 @@ update: python-env collect
# Generate full fields for each beat. Create fields.generated.yml in case it does not exist yet.
# Most beats have their own way of generating it.
test -s _meta/fields.generated.yml || cp _meta/fields.yml _meta/fields.generated.yml
cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > _meta/fields.full.generated.yml
cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > fields.yml
# Revert changes to libbeat fields as it otherwise contains duplicates in full
cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml > ${ES_BEATS}/libbeat/_meta/fields.full.generated.yml
cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml > ${ES_BEATS}/libbeat/fields.yml

# Update docs
mkdir -p docs
. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_NAME} ${ES_BEATS}

# Generate index templates
go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml
go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml
go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml
go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml

# Generate index-pattern
echo "Generate index pattern"
Expand Down
2 changes: 1 addition & 1 deletion libbeat/scripts/generate_fields_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def fields_to_asciidoc(input, output, beat):
beat_name = args.beatname
es_beats = args.es_beats

fields_yml = beat_path + "/_meta/fields.full.generated.yml"
fields_yml = beat_path + "/fields.yml"

# Read fields.yml
with open(fields_yml) as f:
Expand Down
2 changes: 1 addition & 1 deletion libbeat/scripts/generate_index_pattern.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def get_index_pattern_name(index):

args = parser.parse_args()

fields_yml = args.beat + "/_meta/fields.full.generated.yml"
fields_yml = args.beat + "/fields.yml"

# generate the index-pattern content
with open(fields_yml, 'r') as f:
Expand Down
64 changes: 40 additions & 24 deletions libbeat/template/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package template
import (
"fmt"

"github.com/elastic/beats/libbeat/beat"
"github.com/elastic/beats/libbeat/common"
"github.com/elastic/go-ucfg/yaml"
)
Expand All @@ -17,44 +16,61 @@ var (
dynamicTemplates []common.MapStr
)

// GetTemplate creates a template based on the given inputs
func GetTemplate(version string, beatName string, files []string) (common.MapStr, error) {
type Template struct {
index string
beatVersion Version
esVersion Version
}

beatVersion := beat.GetDefaultVersion()
// New creates a new template instance
func New(beatVersion string, esVersion string, index string) (*Template, error) {

bV, err := NewVersion(beatVersion)
if err != nil {
return nil, err
}

// In case no esVersion is set, it is assumed the same as beat version
if version == "" {
version = beatVersion
if esVersion == "" {
esVersion = beatVersion
}

esVersion, err := NewVersion(version)
esV, err := NewVersion(esVersion)
if err != nil {
return nil, err
}

fields := Fields{}
return &Template{
index: index,
beatVersion: *bV,
esVersion: *esV,
}, nil

for _, file := range files {
f, err := loadYaml(file)
if err != nil {
return nil, err
}
fields = append(fields, f...)
}

// Load the given input and generates the input based on it
func (t *Template) Load(file string) (common.MapStr, error) {

fields, err := loadYaml(file)
if err != nil {
return nil, err
}

// Start processing at the root
properties := fields.process("", *esVersion)

indexPattern := fmt.Sprintf("%s-%s-*", beatName, beatVersion)
output := createTemplate(properties, beatVersion, *esVersion, indexPattern, dynamicTemplates)
properties := fields.process("", t.esVersion)
output := t.generate(properties, dynamicTemplates)

return output, nil
}

// GetName returns the name of the template which is {index}-{version}
func (t *Template) GetName() string {
return fmt.Sprintf("%s-%s", t.index, t.beatVersion.String())
}

// createTemplate creates the full template
// generate generates the full template
// The default values are taken from the default variable.
func createTemplate(properties common.MapStr, version string, esVersion Version, indexPattern string, dynamicTemplates []common.MapStr) common.MapStr {
func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr {

// Add base dynamic template
var dynamicTemplateBase = common.MapStr{
Expand All @@ -67,7 +83,7 @@ func createTemplate(properties common.MapStr, version string, esVersion Version,
},
}

if esVersion.IsMajor(2) {
if t.esVersion.IsMajor(2) {
dynamicTemplateBase.Put("strings_as_keyword.mapping.type", "string")
dynamicTemplateBase.Put("strings_as_keyword.mapping.index", "not_analyzed")
}
Expand All @@ -79,7 +95,7 @@ func createTemplate(properties common.MapStr, version string, esVersion Version,
"mappings": common.MapStr{
"_default_": common.MapStr{
"_meta": common.MapStr{
"version": version,
"version": t.beatVersion.String(),
},
"date_detection": defaultDateDetection,
"dynamic_templates": dynamicTemplates,
Expand All @@ -90,10 +106,10 @@ func createTemplate(properties common.MapStr, version string, esVersion Version,
"settings": common.MapStr{
"index.refresh_interval": "5s",
},
"template": indexPattern,
"template": t.GetName() + "-*",
}

if esVersion.IsMajor(2) {
if t.esVersion.IsMajor(2) {
basicStructure.Put("mappings._default_._all.norms.enabled", false)
} else {
// Metricbeat exceeds the default of 1000 fields
Expand Down

0 comments on commit e29ce83

Please sign in to comment.