diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 233432fda3e..d0a0d1b16f1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -51,7 +51,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update replicaset group to apps/v1 {pull}15854[15802] - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] - Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937} -- Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038] +- Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] *Auditbeat* diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index 1191b1ae011..ad1ff3e45de 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -79,7 +79,7 @@ func newKafkaClient( hosts: hosts, topic: topic, key: key, - index: index, + index: strings.ToLower(index), codec: writer, config: *cfg, } diff --git a/libbeat/outputs/logstash/config.go b/libbeat/outputs/logstash/config.go index 598413f5814..6d2a30d39ea 100644 --- a/libbeat/outputs/logstash/config.go +++ b/libbeat/outputs/logstash/config.go @@ -18,6 +18,7 @@ package logstash import ( + "strings" "time" "github.com/elastic/beats/libbeat/beat" @@ -80,7 +81,7 @@ func readConfig(cfg *common.Config, info beat.Info) (*Config, error) { } if c.Index == "" { - c.Index = info.IndexPrefix + c.Index = strings.ToLower(info.IndexPrefix) } return &c, nil diff --git a/libbeat/outputs/logstash/enc.go b/libbeat/outputs/logstash/enc.go index b9fa409b6e9..fb42626ec5c 100644 --- a/libbeat/outputs/logstash/enc.go +++ b/libbeat/outputs/logstash/enc.go @@ -18,6 +18,8 @@ package logstash import ( + "strings" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/outputs/codec/json" ) @@ -27,6 +29,7 @@ func makeLogstashEventEncoder(info beat.Info, escapeHTML bool, index string) fun Pretty: false, EscapeHTML: escapeHTML, }) + index = strings.ToLower(index) return func(event interface{}) (d []byte, err error) { d, err = enc.Encode(index, event.(*beat.Event)) if err != nil { diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 8941984b922..6aca15337fe 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -92,7 +92,8 @@ func esConnect(t *testing.T, index string) *esConnection { host := getElasticsearchHost() indexFmt := fmtstr.MustCompileEvent(fmt.Sprintf("%s-%%{+yyyy.MM.dd}", index)) - indexSel := outil.MakeSelector(outil.FmtSelectorExpr(indexFmt, "")) + indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "") + indexSel := outil.MakeSelector(indexFmtExpr) index, _ = indexSel.Select(&beat.Event{ Timestamp: ts, }) diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index d6ff4931b64..d06ee4e3209 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -19,6 +19,7 @@ package outil import ( "fmt" + "strings" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" @@ -26,10 +27,14 @@ import ( "github.com/elastic/beats/libbeat/conditions" ) +// Selector is used to produce a string based on the contents of a Beats event. +// A selector supports multiple rules that need to be configured. type Selector struct { sel SelectorExpr } +// Settings configures how BuildSelectorFromConfig creates a Selector from +// a given configuration object. type Settings struct { // single selector key and default option keyword Key string @@ -44,6 +49,8 @@ type Settings struct { FailEmpty bool } +// SelectorExpr represents an expression object that can be composed with other +// expressions in order to build a Selector. type SelectorExpr interface { sel(evt *beat.Event) (string, error) } @@ -76,6 +83,7 @@ type mapSelector struct { var nilSelector SelectorExpr = &emptySelector{} +// MakeSelector creates a selector from a set of selector expressions. func MakeSelector(es ...SelectorExpr) Selector { switch len(es) { case 0: @@ -95,10 +103,12 @@ func (s Selector) Select(evt *beat.Event) (string, error) { return s.sel.sel(evt) } +// IsEmpty checks if the selector is not configured and will always return an empty string. func (s Selector) IsEmpty() bool { return s.sel == nilSelector || s.sel == nil } +// IsConst checks if the selector will always return the same string. func (s Selector) IsConst() bool { if s.sel == nilSelector { return true @@ -108,6 +118,7 @@ func (s Selector) IsConst() bool { return ok } +// BuildSelectorFromConfig creates a selector from a configuration object. func BuildSelectorFromConfig( cfg *common.Config, settings Settings, @@ -156,17 +167,13 @@ func BuildSelectorFromConfig( return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } - if fmtstr.IsConst() { - str, err := fmtstr.Run(nil) - if err != nil { - return Selector{}, err - } + fmtsel, err := FmtSelectorExpr(fmtstr, "") + if err != nil { + return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) + } - if str != "" { - sel = append(sel, ConstSelectorExpr(str)) - } - } else { - sel = append(sel, FmtSelectorExpr(fmtstr, "")) + if fmtsel != nilSelector { + sel = append(sel, fmtsel) } } @@ -183,22 +190,44 @@ func BuildSelectorFromConfig( return MakeSelector(sel...), nil } +// EmptySelectorExpr create a selector expression that returns an empty string. func EmptySelectorExpr() SelectorExpr { return nilSelector } +// ConstSelectorExpr creates a selector expression that always returns the configured string. func ConstSelectorExpr(s string) SelectorExpr { - return &constSelector{s} + if s == "" { + return EmptySelectorExpr() + } + return &constSelector{strings.ToLower(s)} } -func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) SelectorExpr { - return &fmtSelector{*fmt, fallback} +// FmtSelectorExpr creates a selector expression using a format string. If the +// event can not be applied the default fallback constant string will be returned. +func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorExpr, error) { + if fmt.IsConst() { + str, err := fmt.Run(nil) + if err != nil { + return nil, err + } + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &fmtSelector{*fmt, strings.ToLower(fallback)}, nil } +// ConcatSelectorExpr combines multiple expressions that are run one after the other. +// The first expression that returns a string wins. func ConcatSelectorExpr(s ...SelectorExpr) SelectorExpr { return &listSelector{s} } +// ConditionalSelectorExpr executes the given expression only if the event +// matches the given condition. func ConditionalSelectorExpr( s SelectorExpr, cond conditions.Condition, @@ -206,12 +235,39 @@ func ConditionalSelectorExpr( return &condSelector{s, cond} } +// LookupSelectorExpr replaces the produced string with an table entry. +// If there is no entry in the table the default fallback string will be reported. func LookupSelectorExpr( - s SelectorExpr, + evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, -) SelectorExpr { - return &mapSelector{s, fallback, table} +) (SelectorExpr, error) { + if evtfmt.IsConst() { + str, err := evtfmt.Run(nil) + if err != nil { + return nil, err + } + + str = table[strings.ToLower(str)] + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &mapSelector{ + from: &fmtSelector{f: *evtfmt}, + to: table, + otherwise: fallback, + }, nil +} + +func lowercaseTable(table map[string]string) map[string]string { + tmp := make(map[string]string, len(table)) + for k, v := range table { + tmp[strings.ToLower(k)] = strings.ToLower(v) + } + return tmp } func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { @@ -239,7 +295,7 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - otherwise = tmp + otherwise = strings.ToLower(tmp) } // 3. extract optional `mapping` @@ -276,45 +332,14 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { // 5. build selector from available fields var sel SelectorExpr if len(mapping.Table) > 0 { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - str = mapping.Table[str] - if str == "" { - str = otherwise - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = &mapSelector{ - from: FmtSelectorExpr(evtfmt, ""), - to: mapping.Table, - otherwise: otherwise, - } - } + sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) } else { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = FmtSelectorExpr(evtfmt, otherwise) - } + sel, err = FmtSelectorExpr(evtfmt, otherwise) } + if err != nil { + return nil, err + } + if cond != nil && sel != nilSelector { sel = ConditionalSelectorExpr(sel, cond) } @@ -363,7 +388,7 @@ func (s *fmtSelector) sel(evt *beat.Event) (string, error) { if n == "" { return s.otherwise, nil } - return n, nil + return strings.ToLower(n), nil } func (s *mapSelector) sel(evt *beat.Event) (string, error) { diff --git a/libbeat/outputs/outil/select_test.go b/libbeat/outputs/outil/select_test.go index d4093ed5660..f6e837966b2 100644 --- a/libbeat/outputs/outil/select_test.go +++ b/libbeat/outputs/outil/select_test.go @@ -31,74 +31,92 @@ import ( type node map[string]interface{} func TestSelector(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string event common.MapStr expected string }{ - { - "constant key", + "constant key": { `key: value`, common.MapStr{}, "value", }, - { - "format string key", + "lowercase constant key": { + `key: VaLuE`, + common.MapStr{}, + "value", + }, + "format string key": { `key: '%{[key]}'`, common.MapStr{"key": "value"}, "value", }, - { - "key with empty keys", + "lowercase format string key": { + `key: '%{[key]}'`, + common.MapStr{"key": "VaLuE"}, + "value", + }, + "key with empty keys": { `{key: value, keys: }`, common.MapStr{}, "value", }, - { - "constant in multi key", + "lowercase key with empty keys": { + `{key: vAlUe, keys: }`, + common.MapStr{}, + "value", + }, + "constant in multi key": { `keys: [key: 'value']`, common.MapStr{}, "value", }, - { - "format string in multi key", + "format string in multi key": { `keys: [key: '%{[key]}']`, common.MapStr{"key": "value"}, "value", }, - { - "missing format string key with default in rule", + "missing format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{}, "value", }, - { - "empty format string key with default in rule", + "lowercase missing format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAlUe`, + common.MapStr{}, + "value", + }, + "empty format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{"key": ""}, "value", }, - { - "missing format string key with constant in next rule", + "lowercase empty format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAluE`, + common.MapStr{"key": ""}, + "value", + }, + "missing format string key with constant in next rule": { `keys: - key: '%{[key]}' - key: value`, common.MapStr{}, "value", }, - { - "missing format string key with constant in top-level rule", + "missing format string key with constant in top-level rule": { `{ key: value, keys: [key: '%{[key]}']}`, common.MapStr{}, "value", }, - { - "apply mapping", + "apply mapping": { `keys: - key: '%{[key]}' mappings: @@ -106,8 +124,15 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping with default on empty key", + "lowercase applied mapping": { + `keys: + - key: '%{[key]}' + mappings: + v: vAlUe`, + common.MapStr{"key": "v"}, + "value", + }, + "apply mapping with default on empty key": { `keys: - key: '%{[key]}' default: value @@ -116,8 +141,16 @@ func TestSelector(t *testing.T) { common.MapStr{"key": ""}, "value", }, - { - "apply mapping with default on empty lookup", + "lowercase apply mapping with default on empty key": { + `keys: + - key: '%{[key]}' + default: vAluE + mappings: + v: 'v'`, + common.MapStr{"key": ""}, + "value", + }, + "apply mapping with default on empty lookup": { `keys: - key: '%{[key]}' default: value @@ -126,8 +159,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping without match", + "apply mapping without match": { `keys: - key: '%{[key]}' mappings: @@ -136,8 +168,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "x"}, "value", }, - { - "mapping with constant key", + "mapping with constant key": { `keys: - key: k mappings: @@ -145,8 +176,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key", + "mapping with missing constant key": { `keys: - key: unknown mappings: {k: wrong} @@ -154,8 +184,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key, but default", + "mapping with missing constant key, but default": { `keys: - key: unknown default: value @@ -163,16 +192,14 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "matching condition", + "matching condition": { `keys: - key: value when.equals.test: test`, common.MapStr{"test": "test"}, "value", }, - { - "failing condition", + "failing condition": { `keys: - key: wrong when.equals.test: test @@ -182,113 +209,98 @@ func TestSelector(t *testing.T) { }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) - - yaml := strings.Replace(test.config, "\t", " ", -1) - cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") - if err != nil { - t.Errorf("YAML parse error: %v\n%v", err, yaml) - continue - } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + yaml := strings.Replace(test.config, "\t", " ", -1) + cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") + if err != nil { + t.Fatalf("YAML parse error: %v\n%v", err, yaml) + } - sel, err := BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, - }) - if err != nil { - t.Error(err) - continue - } + sel, err := BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) + if err != nil { + t.Fatal(err) + } - event := beat.Event{ - Timestamp: time.Now(), - Fields: test.event, - } - actual, err := sel.Select(&event) - if err != nil { - t.Error(err) - continue - } + event := beat.Event{ + Timestamp: time.Now(), + Fields: test.event, + } + actual, err := sel.Select(&event) + if err != nil { + t.Fatal(err) + } - assert.Equal(t, test.expected, actual) + assert.Equal(t, test.expected, actual) + }) } } func TestSelectorInitFail(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string }{ - { - "keys missing", + "keys missing": { `test: no key`, }, - { - "invalid keys type", + "invalid keys type": { `keys: 5`, }, - { - "invaid keys element type", + "invaid keys element type": { `keys: [5]`, }, - { - "invalid key type", + "invalid key type": { `key: {}`, }, - { - "missing key in list", + "missing key in list": { `keys: [default: value]`, }, - { - "invalid key type in list", + "invalid key type in list": { `keys: [key: {}]`, }, - { - "fail on invalid format string", + "fail on invalid format string": { `key: '%{[abc}'`, }, - { - "fail on invalid format string in list", + "fail on invalid format string in list": { `keys: [key: '%{[abc}']`, }, - { - "default value type mismatch", + "default value type mismatch": { `keys: [{key: ok, default: {}}]`, }, - { - "mappings type mismatch", + "mappings type mismatch": { `keys: - key: '%{[k]}' mappings: {v: {}}`, }, - { - "condition empty", + "condition empty": { `keys: - key: value when:`, }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) + for name, test := range tests { + t.Run(name, func(t *testing.T) { + cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") + if err != nil { + t.Fatal(err) + } - cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") - if err != nil { - t.Error(err) - continue - } + _, err = BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) - _, err = BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, + assert.Error(t, err) + t.Log(err) }) - assert.Error(t, err) - t.Log(err) } } diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 7238ba367fd..1764d9e60f9 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -21,6 +21,7 @@ import ( "errors" "regexp" "strconv" + "strings" "time" "github.com/garyburd/redigo/redis" @@ -77,7 +78,7 @@ func newClient( observer: observer, timeout: timeout, password: pass, - index: index, + index: strings.ToLower(index), db: db, dataType: dt, key: key,