Skip to content

Commit

Permalink
Align kibana index generation to beats standard. (#5290)
Browse files Browse the repository at this point in the history 
* Move generator script to dev-tools.
* Add NewGenerator method which returns struct instead of init method.
  • Loading branch information
@simitt @ruflin
simitt authored and ruflin committed Oct 2, 2017
1 parent 3c075f7 commit ffca490
Show file tree
Hide file tree
Showing 8 changed files with 184 additions and 328 deletions.
49 changes: 49 additions & 0 deletions dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package main

import (
"flag"
"fmt"
"os"

"github.com/elastic/beats/libbeat/kibana"
"github.com/elastic/beats/libbeat/version"
)

func main() {
beatVersion := version.GetDefaultVersion()
index := flag.String("index", "", "The name of the index pattern. (required)")
beatName := flag.String("beat-name", "", "The name of the beat. (required)")
beatDir := flag.String("beat-dir", "", "The local beat directory. (required)")
version := flag.String("version", beatVersion, "The beat version.")
flag.Parse()

if *index == "" {
fmt.Fprint(os.Stderr, "The name of the index pattern msut be set.")
os.Exit(1)
}

if *beatName == "" {
fmt.Fprint(os.Stderr, "The name of the beat must be set.")
os.Exit(1)
}

if *beatDir == "" {
fmt.Fprint(os.Stderr, "The beat directory must be set.")
os.Exit(1)
}

indexPatternGenerator, err := kibana.NewGenerator(*index, *beatName, *beatDir, *version)
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}

pattern, err := indexPatternGenerator.Generate()
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
for _, p := range pattern {
fmt.Fprintf(os.Stdout, "-- The index pattern was created under %v\n", p)
}
}
72 changes: 33 additions & 39 deletions libbeat/kibana/index.go → libbeat/kibana/index_pattern_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package kibana

import (
"encoding/json"
"errors"
"io/ioutil"
"os"
"path/filepath"
Expand All @@ -11,43 +10,56 @@ import (
"github.com/elastic/beats/libbeat/common"
)

type Index struct {
Version string
IndexName string
BeatDir string
BeatName string

type IndexPatternGenerator struct {
indexName string
version string
fieldsYaml string
targetDirDefault string
targetDir5x string
targetFilename string
}

// Create the Index-Pattern for Kibana for 5.x and default.
func (i *Index) Create() ([]string, error) {
indices := []string{}
// Create an instance of the Kibana Index Pattern Generator
func NewGenerator(indexName, beatName, beatDir, version string) (*IndexPatternGenerator, error) {
beatName = clean(beatName)

err := i.init()
if err != nil {
fieldsYaml := filepath.Join(beatDir, "fields.yml")
if _, err := os.Stat(fieldsYaml); err != nil {
return nil, err
}

return &IndexPatternGenerator{
indexName: indexName,
version: version,
fieldsYaml: fieldsYaml,
targetDirDefault: createTargetDir(beatDir, "default"),
targetDir5x: createTargetDir(beatDir, "5.x"),
targetFilename: beatName + ".json",
}, nil
}

// Create the Index-Pattern for Kibana for 5.x and default.
func (i *IndexPatternGenerator) Generate() ([]string, error) {
var indices []string

commonFields, err := common.LoadFieldsYaml(i.fieldsYaml)
if err != nil {
return nil, err
}
transformed := TransformFields("@timestamp", i.IndexName, commonFields)
transformer := NewTransformer("@timestamp", i.indexName, commonFields)
transformed := transformer.TransformFields()

if fieldsBytes, err := json.Marshal(transformed["fields"]); err != nil {
fieldsBytes, err := json.Marshal(transformed["fields"])
if err != nil {
return nil, err
} else {
transformed["fields"] = string(fieldsBytes)
}
if fieldFormatBytes, err := json.Marshal(transformed["fieldFormatMap"]); err != nil {
transformed["fields"] = string(fieldsBytes)

fieldFormatBytes, err := json.Marshal(transformed["fieldFormatMap"])
if err != nil {
return nil, err
} else {
transformed["fieldFormatMap"] = string(fieldFormatBytes)
}
transformed["fieldFormatMap"] = string(fieldFormatBytes)

file5x := filepath.Join(i.targetDir5x, i.targetFilename)
err = dumpToFile(file5x, transformed)
Expand All @@ -57,11 +69,11 @@ func (i *Index) Create() ([]string, error) {
indices = append(indices, file5x)

out := common.MapStr{
"version": i.Version,
"version": i.version,
"objects": []common.MapStr{
common.MapStr{
"type": "index-pattern",
"id": i.IndexName,
"id": i.indexName,
"version": 1,
"attributes": transformed,
},
Expand All @@ -77,24 +89,6 @@ func (i *Index) Create() ([]string, error) {
return indices, nil
}

func (i *Index) init() error {
if i.Version == "" || i.IndexName == "" || i.BeatDir == "" || i.BeatName == "" {
return errors.New("RequiredParams: Version, IndexName, BeatDir and BeatName")
}
i.BeatName = clean(i.BeatName)

i.fieldsYaml = filepath.Join(i.BeatDir, "fields.yml")
if _, err := os.Stat(i.fieldsYaml); err != nil {
return err
}

i.targetDirDefault = createTargetDir(i.BeatDir, "default")
i.targetDir5x = createTargetDir(i.BeatDir, "5.x")
i.targetFilename = i.BeatName + ".json"

return nil
}

func clean(name string) string {
reg := regexp.MustCompile("[^a-zA-Z0-9_]+")
return reg.ReplaceAllString(name, "")
Expand Down
82 changes: 51 additions & 31 deletions libbeat/kibana/index_test.go → ...at/kibana/index_pattern_generator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,49 +14,32 @@ import (
"github.com/elastic/beats/libbeat/common"
)

func TestAllArgsSet(t *testing.T) {
func TestNewGenerator(t *testing.T) {
beatDir := tmpPath()
defer teardown(beatDir)

tests := []struct {
Index Index
}{
{Index: Index{IndexName: "beat-index", BeatDir: beatDir, BeatName: "mybeat."}},
{Index: Index{Version: "6.0", BeatDir: beatDir, BeatName: "mybeat."}},
{Index: Index{Version: "6.0", IndexName: "beat-index", BeatName: "mybeat."}},
{Index: Index{Version: "6.0", IndexName: "beat-index", BeatDir: beatDir}},
}
for idx, test := range tests {
err := test.Index.init()
msg := fmt.Sprintf("(%v): Should have raised error", idx)
assert.Error(t, err, msg)
}
}

func TestInit(t *testing.T) {
beatDir := tmpPath()
defer teardown(beatDir)
// checks for fields.yml
idx := Index{Version: "7.0", IndexName: "beat-index", BeatDir: filepath.Join(beatDir, "notexistent"), BeatName: "mybeat."}
err := idx.init()
generator, err := NewGenerator("beat-index", "mybeat.", filepath.Join(beatDir, "notexistent"), "7.0")
assert.Error(t, err)

idx = Index{Version: "7.0", IndexName: "beat-index", BeatDir: beatDir, BeatName: "mybeat."}
err = idx.init()
generator, err = NewGenerator("beat-index", "mybeat.", beatDir, "7.0")
assert.NoError(t, err)
assert.Equal(t, "7.0", generator.version)
assert.Equal(t, "beat-index", generator.indexName)
assert.Equal(t, filepath.Join(beatDir, "fields.yml"), generator.fieldsYaml)

// creates file dir and sets name
expectedDir := filepath.Join(beatDir, "_meta/kibana/default/index-pattern")
assert.Equal(t, expectedDir, idx.targetDirDefault)
_, err = os.Stat(idx.targetDirDefault)
assert.Equal(t, expectedDir, generator.targetDirDefault)
_, err = os.Stat(generator.targetDirDefault)
assert.NoError(t, err)

expectedDir = filepath.Join(beatDir, "_meta/kibana/5.x/index-pattern")
assert.Equal(t, expectedDir, idx.targetDir5x)
_, err = os.Stat(idx.targetDir5x)
assert.Equal(t, expectedDir, generator.targetDir5x)
_, err = os.Stat(generator.targetDir5x)
assert.NoError(t, err)

assert.Equal(t, "mybeat.json", idx.targetFilename)
assert.Equal(t, "mybeat.json", generator.targetFilename)
}

func TestCleanName(t *testing.T) {
Expand All @@ -75,12 +58,49 @@ func TestCleanName(t *testing.T) {
}
}

func TestDefault(t *testing.T) {
func TestGenerateFieldsYaml(t *testing.T) {
beatDir := tmpPath()
defer teardown(beatDir)
generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1")
_, err = generator.Generate()
assert.NoError(t, err)

generator.fieldsYaml = ""
_, err = generator.Generate()
assert.Error(t, err)
}

func TestDumpToFile5x(t *testing.T) {
beatDir := tmpPath()
defer teardown(beatDir)
generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1")
_, err = generator.Generate()
assert.NoError(t, err)

generator.targetDir5x = "./non-existing/something"
_, err = generator.Generate()
assert.Error(t, err)
}

index := Index{Version: "7.0.0-alpha1", IndexName: "metricbeat-*", BeatDir: beatDir, BeatName: "metric beat !"}
index.Create()
func TestDumpToFileDefault(t *testing.T) {
beatDir := tmpPath()
defer teardown(beatDir)
generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1")
_, err = generator.Generate()
assert.NoError(t, err)

generator.targetDirDefault = "./non-existing/something"
_, err = generator.Generate()
assert.Error(t, err)
}

func TestGenerate(t *testing.T) {
beatDir := tmpPath()
defer teardown(beatDir)
generator, err := NewGenerator("metricbeat-*", "metric beat ?!", beatDir, "7.0.0-alpha1")
pattern, err := generator.Generate()
assert.NoError(t, err)
assert.Equal(t, 2, len(pattern))

tests := []map[string]string{
{"existing": "metricbeat-5x-old.json", "created": "_meta/kibana/5.x/index-pattern/metricbeat.json"},
Expand Down
58 changes: 37 additions & 21 deletions libbeat/kibana/transform.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,46 +7,62 @@ import (
"github.com/elastic/beats/libbeat/common"
)

func TransformFields(timeFieldName string, title string, commonFields common.Fields) common.MapStr {
fields := []common.MapStr{}
fieldFormatMap := common.MapStr{}
keys := common.MapStr{}
type Transformer struct {
fields common.Fields
transformedFields []common.MapStr
transformedFieldFormatMap common.MapStr
timeFieldName string
title string
keys common.MapStr
}

func NewTransformer(timeFieldName, title string, fields common.Fields) *Transformer {
return &Transformer{
fields: fields,
timeFieldName: timeFieldName,
title: title,
transformedFields: []common.MapStr{},
transformedFieldFormatMap: common.MapStr{},
keys: common.MapStr{},
}
}

transformFields(keys, commonFields, &fields, fieldFormatMap, "")
func (t *Transformer) TransformFields() common.MapStr {
t.transformFields(t.fields, "")

// add some meta fields
truthy := true
falsy := false
add(common.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap)
add(common.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy}, &fields, fieldFormatMap)
add(common.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap)
add(common.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy}, &fields, fieldFormatMap)
t.add(common.Field{Path: "_id", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy})
t.add(common.Field{Path: "_type", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &truthy, Aggregatable: &truthy})
t.add(common.Field{Path: "_index", Type: "keyword", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy})
t.add(common.Field{Path: "_score", Type: "integer", Index: &falsy, Analyzed: &falsy, DocValues: &falsy, Searchable: &falsy, Aggregatable: &falsy})

return common.MapStr{
"timeFieldName": timeFieldName,
"title": title,
"fields": fields,
"fieldFormatMap": fieldFormatMap,
"timeFieldName": t.timeFieldName,
"title": t.title,
"fields": t.transformedFields,
"fieldFormatMap": t.transformedFieldFormatMap,
}
}

func transformFields(keys common.MapStr, commonFields common.Fields, fields *[]common.MapStr, fieldFormatMap common.MapStr, path string) {
func (t *Transformer) transformFields(commonFields common.Fields, path string) {
for _, f := range commonFields {
f.Path = f.Name
if path != "" {
f.Path = path + "." + f.Name
}

if keys[f.Path] != nil {
if t.keys[f.Path] != nil {
msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.", f.Path)
panic(errors.New(msg))
}

if f.Type == "group" {
transformFields(keys, f.Fields, fields, fieldFormatMap, f.Path)
t.transformFields(f.Fields, f.Path)
} else {
// set default values (as done in python script)
keys[f.Path] = true
t.keys[f.Path] = true

truthy := true
falsy := false
Expand All @@ -55,16 +71,16 @@ func transformFields(keys common.MapStr, commonFields common.Fields, fields *[]c
f.DocValues = &truthy
f.Searchable = &truthy
f.Aggregatable = &truthy
add(f, fields, fieldFormatMap)
t.add(f)
}
}
}

func add(f common.Field, fields *[]common.MapStr, fieldFormatMap common.MapStr) {
func (t *Transformer) add(f common.Field) {
field, fieldFormat := transformField(f)
*fields = append(*fields, field)
t.transformedFields = append(t.transformedFields, field)
if fieldFormat != nil {
fieldFormatMap[field["name"].(string)] = fieldFormat
t.transformedFieldFormatMap[field["name"].(string)] = fieldFormat
}

}
Expand Down
Loading

0 comments on commit ffca490

Please sign in to comment.