From e14e62014482ee19cbdf8aebded78cbbeca08b46 Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 4 Feb 2020 19:10:02 +0100 Subject: [PATCH 1/8] Index names must be lowercase When indexing into Elasticsearch index names must always be lowercase. If the index or indices setting are configured to produce non-lowercase strings (e.g. by extracting part of the index name from the event contents), we need to normalize them to be lowercase. This change ensure that index names are always converted to lowercase. Static strings are converted to lowercase upfront, while dynamic strings will be post-processed. --- libbeat/outputs/outil/select.go | 77 +++++----- libbeat/outputs/outil/select_test.go | 222 ++++++++++++++------------- 2 files changed, 159 insertions(+), 140 deletions(-) diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index d6ff4931b64..5af6954a850 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -19,6 +19,7 @@ package outil import ( "fmt" + "strings" "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" @@ -188,11 +189,14 @@ func EmptySelectorExpr() SelectorExpr { } func ConstSelectorExpr(s string) SelectorExpr { - return &constSelector{s} + if s == "" { + return nilSelector + } + return &constSelector{strings.ToLower(s)} } func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) SelectorExpr { - return &fmtSelector{*fmt, fallback} + return &fmtSelector{*fmt, strings.ToLower(fallback)} } func ConcatSelectorExpr(s ...SelectorExpr) SelectorExpr { @@ -207,11 +211,36 @@ func ConditionalSelectorExpr( } func LookupSelectorExpr( - s SelectorExpr, + evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, -) SelectorExpr { - return &mapSelector{s, fallback, table} +) (SelectorExpr, error) { + if evtfmt.IsConst() { + str, err := evtfmt.Run(nil) + if err != nil { + return nil, err + } + + str = table[strings.ToLower(str)] + if str == "" { + str = fallback + } + return ConstSelectorExpr(str), nil + } + + return &mapSelector{ + from: FmtSelectorExpr(evtfmt, ""), + to: table, + otherwise: fallback, + }, nil +} + +func lowercaseTable(table map[string]string) map[string]string { + tmp := make(map[string]string, len(table)) + for k, v := range table { + tmp[strings.ToLower(k)] = strings.ToLower(v) + } + return tmp } func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { @@ -239,7 +268,7 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - otherwise = tmp + otherwise = strings.ToLower(tmp) } // 3. extract optional `mapping` @@ -276,28 +305,10 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { // 5. build selector from available fields var sel SelectorExpr if len(mapping.Table) > 0 { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - - str = mapping.Table[str] - if str == "" { - str = otherwise - } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } - } else { - sel = &mapSelector{ - from: FmtSelectorExpr(evtfmt, ""), - to: mapping.Table, - otherwise: otherwise, - } + var err error + sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) + if err != nil { + return nil, err } } else { if evtfmt.IsConst() { @@ -305,16 +316,12 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - - if str == "" { - sel = nilSelector - } else { - sel = ConstSelectorExpr(str) - } + sel = ConstSelectorExpr(str) } else { sel = FmtSelectorExpr(evtfmt, otherwise) } } + if cond != nil && sel != nilSelector { sel = ConditionalSelectorExpr(sel, cond) } @@ -363,7 +370,7 @@ func (s *fmtSelector) sel(evt *beat.Event) (string, error) { if n == "" { return s.otherwise, nil } - return n, nil + return strings.ToLower(n), nil } func (s *mapSelector) sel(evt *beat.Event) (string, error) { diff --git a/libbeat/outputs/outil/select_test.go b/libbeat/outputs/outil/select_test.go index d4093ed5660..f6e837966b2 100644 --- a/libbeat/outputs/outil/select_test.go +++ b/libbeat/outputs/outil/select_test.go @@ -31,74 +31,92 @@ import ( type node map[string]interface{} func TestSelector(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string event common.MapStr expected string }{ - { - "constant key", + "constant key": { `key: value`, common.MapStr{}, "value", }, - { - "format string key", + "lowercase constant key": { + `key: VaLuE`, + common.MapStr{}, + "value", + }, + "format string key": { `key: '%{[key]}'`, common.MapStr{"key": "value"}, "value", }, - { - "key with empty keys", + "lowercase format string key": { + `key: '%{[key]}'`, + common.MapStr{"key": "VaLuE"}, + "value", + }, + "key with empty keys": { `{key: value, keys: }`, common.MapStr{}, "value", }, - { - "constant in multi key", + "lowercase key with empty keys": { + `{key: vAlUe, keys: }`, + common.MapStr{}, + "value", + }, + "constant in multi key": { `keys: [key: 'value']`, common.MapStr{}, "value", }, - { - "format string in multi key", + "format string in multi key": { `keys: [key: '%{[key]}']`, common.MapStr{"key": "value"}, "value", }, - { - "missing format string key with default in rule", + "missing format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{}, "value", }, - { - "empty format string key with default in rule", + "lowercase missing format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAlUe`, + common.MapStr{}, + "value", + }, + "empty format string key with default in rule": { `keys: - key: '%{[key]}' default: value`, common.MapStr{"key": ""}, "value", }, - { - "missing format string key with constant in next rule", + "lowercase empty format string key with default in rule": { + `keys: + - key: '%{[key]}' + default: vAluE`, + common.MapStr{"key": ""}, + "value", + }, + "missing format string key with constant in next rule": { `keys: - key: '%{[key]}' - key: value`, common.MapStr{}, "value", }, - { - "missing format string key with constant in top-level rule", + "missing format string key with constant in top-level rule": { `{ key: value, keys: [key: '%{[key]}']}`, common.MapStr{}, "value", }, - { - "apply mapping", + "apply mapping": { `keys: - key: '%{[key]}' mappings: @@ -106,8 +124,15 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping with default on empty key", + "lowercase applied mapping": { + `keys: + - key: '%{[key]}' + mappings: + v: vAlUe`, + common.MapStr{"key": "v"}, + "value", + }, + "apply mapping with default on empty key": { `keys: - key: '%{[key]}' default: value @@ -116,8 +141,16 @@ func TestSelector(t *testing.T) { common.MapStr{"key": ""}, "value", }, - { - "apply mapping with default on empty lookup", + "lowercase apply mapping with default on empty key": { + `keys: + - key: '%{[key]}' + default: vAluE + mappings: + v: 'v'`, + common.MapStr{"key": ""}, + "value", + }, + "apply mapping with default on empty lookup": { `keys: - key: '%{[key]}' default: value @@ -126,8 +159,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "v"}, "value", }, - { - "apply mapping without match", + "apply mapping without match": { `keys: - key: '%{[key]}' mappings: @@ -136,8 +168,7 @@ func TestSelector(t *testing.T) { common.MapStr{"key": "x"}, "value", }, - { - "mapping with constant key", + "mapping with constant key": { `keys: - key: k mappings: @@ -145,8 +176,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key", + "mapping with missing constant key": { `keys: - key: unknown mappings: {k: wrong} @@ -154,8 +184,7 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "mapping with missing constant key, but default", + "mapping with missing constant key, but default": { `keys: - key: unknown default: value @@ -163,16 +192,14 @@ func TestSelector(t *testing.T) { common.MapStr{}, "value", }, - { - "matching condition", + "matching condition": { `keys: - key: value when.equals.test: test`, common.MapStr{"test": "test"}, "value", }, - { - "failing condition", + "failing condition": { `keys: - key: wrong when.equals.test: test @@ -182,113 +209,98 @@ func TestSelector(t *testing.T) { }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) - - yaml := strings.Replace(test.config, "\t", " ", -1) - cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") - if err != nil { - t.Errorf("YAML parse error: %v\n%v", err, yaml) - continue - } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + yaml := strings.Replace(test.config, "\t", " ", -1) + cfg, err := common.NewConfigWithYAML([]byte(yaml), "test") + if err != nil { + t.Fatalf("YAML parse error: %v\n%v", err, yaml) + } - sel, err := BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, - }) - if err != nil { - t.Error(err) - continue - } + sel, err := BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) + if err != nil { + t.Fatal(err) + } - event := beat.Event{ - Timestamp: time.Now(), - Fields: test.event, - } - actual, err := sel.Select(&event) - if err != nil { - t.Error(err) - continue - } + event := beat.Event{ + Timestamp: time.Now(), + Fields: test.event, + } + actual, err := sel.Select(&event) + if err != nil { + t.Fatal(err) + } - assert.Equal(t, test.expected, actual) + assert.Equal(t, test.expected, actual) + }) } } func TestSelectorInitFail(t *testing.T) { - tests := []struct { - title string + tests := map[string]struct { config string }{ - { - "keys missing", + "keys missing": { `test: no key`, }, - { - "invalid keys type", + "invalid keys type": { `keys: 5`, }, - { - "invaid keys element type", + "invaid keys element type": { `keys: [5]`, }, - { - "invalid key type", + "invalid key type": { `key: {}`, }, - { - "missing key in list", + "missing key in list": { `keys: [default: value]`, }, - { - "invalid key type in list", + "invalid key type in list": { `keys: [key: {}]`, }, - { - "fail on invalid format string", + "fail on invalid format string": { `key: '%{[abc}'`, }, - { - "fail on invalid format string in list", + "fail on invalid format string in list": { `keys: [key: '%{[abc}']`, }, - { - "default value type mismatch", + "default value type mismatch": { `keys: [{key: ok, default: {}}]`, }, - { - "mappings type mismatch", + "mappings type mismatch": { `keys: - key: '%{[k]}' mappings: {v: {}}`, }, - { - "condition empty", + "condition empty": { `keys: - key: value when:`, }, } - for i, test := range tests { - t.Logf("run (%v): %v", i, test.title) + for name, test := range tests { + t.Run(name, func(t *testing.T) { + cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") + if err != nil { + t.Fatal(err) + } - cfg, err := common.NewConfigWithYAML([]byte(test.config), "test") - if err != nil { - t.Error(err) - continue - } + _, err = BuildSelectorFromConfig(cfg, Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + }) - _, err = BuildSelectorFromConfig(cfg, Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, + assert.Error(t, err) + t.Log(err) }) - assert.Error(t, err) - t.Log(err) } } From 749b73a8efa30734eae5f89b4cc004035b56183f Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 4 Feb 2020 19:32:40 +0100 Subject: [PATCH 2/8] update kafka/redis/LS output to guarantee lowercase index --- libbeat/outputs/kafka/client.go | 2 +- libbeat/outputs/logstash/config.go | 3 ++- libbeat/outputs/logstash/enc.go | 3 +++ libbeat/outputs/redis/client.go | 3 ++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/libbeat/outputs/kafka/client.go b/libbeat/outputs/kafka/client.go index 1191b1ae011..ad1ff3e45de 100644 --- a/libbeat/outputs/kafka/client.go +++ b/libbeat/outputs/kafka/client.go @@ -79,7 +79,7 @@ func newKafkaClient( hosts: hosts, topic: topic, key: key, - index: index, + index: strings.ToLower(index), codec: writer, config: *cfg, } diff --git a/libbeat/outputs/logstash/config.go b/libbeat/outputs/logstash/config.go index 598413f5814..6d2a30d39ea 100644 --- a/libbeat/outputs/logstash/config.go +++ b/libbeat/outputs/logstash/config.go @@ -18,6 +18,7 @@ package logstash import ( + "strings" "time" "github.com/elastic/beats/libbeat/beat" @@ -80,7 +81,7 @@ func readConfig(cfg *common.Config, info beat.Info) (*Config, error) { } if c.Index == "" { - c.Index = info.IndexPrefix + c.Index = strings.ToLower(info.IndexPrefix) } return &c, nil diff --git a/libbeat/outputs/logstash/enc.go b/libbeat/outputs/logstash/enc.go index b9fa409b6e9..fb42626ec5c 100644 --- a/libbeat/outputs/logstash/enc.go +++ b/libbeat/outputs/logstash/enc.go @@ -18,6 +18,8 @@ package logstash import ( + "strings" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/outputs/codec/json" ) @@ -27,6 +29,7 @@ func makeLogstashEventEncoder(info beat.Info, escapeHTML bool, index string) fun Pretty: false, EscapeHTML: escapeHTML, }) + index = strings.ToLower(index) return func(event interface{}) (d []byte, err error) { d, err = enc.Encode(index, event.(*beat.Event)) if err != nil { diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 7238ba367fd..1764d9e60f9 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -21,6 +21,7 @@ import ( "errors" "regexp" "strconv" + "strings" "time" "github.com/garyburd/redigo/redis" @@ -77,7 +78,7 @@ func newClient( observer: observer, timeout: timeout, password: pass, - index: index, + index: strings.ToLower(index), db: db, dataType: dt, key: key, From 7ab4af3c13b38c23276d22bec7494ec40a09b0be Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 4 Feb 2020 19:34:22 +0100 Subject: [PATCH 3/8] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 233432fda3e..05340d2f561 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -52,6 +52,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] - Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937} - Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038] +- Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] *Auditbeat* From 6773e45d46129ede960937cc47ab7f588800a919 Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 4 Feb 2020 19:45:52 +0100 Subject: [PATCH 4/8] unexport symbols --- libbeat/outputs/outil/select.go | 58 ++++++++++++++++----------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index 5af6954a850..52f7223dd00 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -28,7 +28,7 @@ import ( ) type Selector struct { - sel SelectorExpr + sel selectorExpr } type Settings struct { @@ -45,18 +45,18 @@ type Settings struct { FailEmpty bool } -type SelectorExpr interface { +type selectorExpr interface { sel(evt *beat.Event) (string, error) } type emptySelector struct{} type listSelector struct { - selectors []SelectorExpr + selectors []selectorExpr } type condSelector struct { - s SelectorExpr + s selectorExpr cond conditions.Condition } @@ -70,21 +70,21 @@ type fmtSelector struct { } type mapSelector struct { - from SelectorExpr + from selectorExpr otherwise string to map[string]string } -var nilSelector SelectorExpr = &emptySelector{} +var nilSelector selectorExpr = &emptySelector{} -func MakeSelector(es ...SelectorExpr) Selector { +func MakeSelector(es ...selectorExpr) Selector { switch len(es) { case 0: return Selector{nilSelector} case 1: return Selector{es[0]} default: - return Selector{ConcatSelectorExpr(es...)} + return Selector{concatSelectorExpr(es...)} } } @@ -113,7 +113,7 @@ func BuildSelectorFromConfig( cfg *common.Config, settings Settings, ) (Selector, error) { - var sel []SelectorExpr + var sel []selectorExpr key := settings.Key multiKey := settings.MultiKey @@ -164,10 +164,10 @@ func BuildSelectorFromConfig( } if str != "" { - sel = append(sel, ConstSelectorExpr(str)) + sel = append(sel, constSelectorExpr(str)) } } else { - sel = append(sel, FmtSelectorExpr(fmtstr, "")) + sel = append(sel, fmtSelectorExpr(fmtstr, "")) } } @@ -184,37 +184,37 @@ func BuildSelectorFromConfig( return MakeSelector(sel...), nil } -func EmptySelectorExpr() SelectorExpr { +func emptySelectorExpr() selectorExpr { return nilSelector } -func ConstSelectorExpr(s string) SelectorExpr { +func constSelectorExpr(s string) selectorExpr { if s == "" { - return nilSelector + return emptySelectorExpr() } return &constSelector{strings.ToLower(s)} } -func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) SelectorExpr { +func fmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) selectorExpr { return &fmtSelector{*fmt, strings.ToLower(fallback)} } -func ConcatSelectorExpr(s ...SelectorExpr) SelectorExpr { +func concatSelectorExpr(s ...selectorExpr) selectorExpr { return &listSelector{s} } -func ConditionalSelectorExpr( - s SelectorExpr, +func conditionalSelectorExpr( + s selectorExpr, cond conditions.Condition, -) SelectorExpr { +) selectorExpr { return &condSelector{s, cond} } -func LookupSelectorExpr( +func lookupSelectorExpr( evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, -) (SelectorExpr, error) { +) (selectorExpr, error) { if evtfmt.IsConst() { str, err := evtfmt.Run(nil) if err != nil { @@ -225,11 +225,11 @@ func LookupSelectorExpr( if str == "" { str = fallback } - return ConstSelectorExpr(str), nil + return constSelectorExpr(str), nil } return &mapSelector{ - from: FmtSelectorExpr(evtfmt, ""), + from: fmtSelectorExpr(evtfmt, ""), to: table, otherwise: fallback, }, nil @@ -243,7 +243,7 @@ func lowercaseTable(table map[string]string) map[string]string { return tmp } -func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { +func buildSingle(cfg *common.Config, key string) (selectorExpr, error) { // TODO: check for unknown fields // 1. extract required key-word handler @@ -303,10 +303,10 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { } // 5. build selector from available fields - var sel SelectorExpr + var sel selectorExpr if len(mapping.Table) > 0 { var err error - sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) + sel, err = lookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) if err != nil { return nil, err } @@ -316,14 +316,14 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - sel = ConstSelectorExpr(str) + sel = constSelectorExpr(str) } else { - sel = FmtSelectorExpr(evtfmt, otherwise) + sel = fmtSelectorExpr(evtfmt, otherwise) } } if cond != nil && sel != nilSelector { - sel = ConditionalSelectorExpr(sel, cond) + sel = conditionalSelectorExpr(sel, cond) } return sel, nil From e77dfc329f0280212f4ecd99f831e5a80bfe1c47 Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 4 Feb 2020 19:57:07 +0100 Subject: [PATCH 5/8] cleanup changelog --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 05340d2f561..d0a0d1b16f1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -51,7 +51,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update replicaset group to apps/v1 {pull}15854[15802] - Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] - Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937} -- Remove superfluous use of number_of_routing_shards setting from the default template. {pull}16038[16038] - Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] *Auditbeat* From 288e07fa354b5e65824be268c200056074a26629 Mon Sep 17 00:00:00 2001 From: urso Date: Tue, 4 Feb 2020 20:24:46 +0100 Subject: [PATCH 6/8] More logic in fmtSelectorExpr --- libbeat/outputs/outil/select.go | 50 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index 52f7223dd00..6ad0ef515dd 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -157,17 +157,13 @@ func BuildSelectorFromConfig( return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } - if fmtstr.IsConst() { - str, err := fmtstr.Run(nil) - if err != nil { - return Selector{}, err - } + fmtsel, err := fmtSelectorExpr(fmtstr, "") + if err != nil { + return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) + } - if str != "" { - sel = append(sel, constSelectorExpr(str)) - } - } else { - sel = append(sel, fmtSelectorExpr(fmtstr, "")) + if fmtsel != nilSelector { + sel = append(sel, fmtsel) } } @@ -195,8 +191,19 @@ func constSelectorExpr(s string) selectorExpr { return &constSelector{strings.ToLower(s)} } -func fmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) selectorExpr { - return &fmtSelector{*fmt, strings.ToLower(fallback)} +func fmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (selectorExpr, error) { + if fmt.IsConst() { + str, err := fmt.Run(nil) + if err != nil { + return nil, err + } + if str == "" { + str = fallback + } + return constSelectorExpr(str), nil + } + + return &fmtSelector{*fmt, strings.ToLower(fallback)}, nil } func concatSelectorExpr(s ...selectorExpr) selectorExpr { @@ -229,7 +236,7 @@ func lookupSelectorExpr( } return &mapSelector{ - from: fmtSelectorExpr(evtfmt, ""), + from: &fmtSelector{f: *evtfmt}, to: table, otherwise: fallback, }, nil @@ -305,21 +312,12 @@ func buildSingle(cfg *common.Config, key string) (selectorExpr, error) { // 5. build selector from available fields var sel selectorExpr if len(mapping.Table) > 0 { - var err error sel, err = lookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) - if err != nil { - return nil, err - } } else { - if evtfmt.IsConst() { - str, err := evtfmt.Run(nil) - if err != nil { - return nil, err - } - sel = constSelectorExpr(str) - } else { - sel = fmtSelectorExpr(evtfmt, otherwise) - } + sel, err = fmtSelectorExpr(evtfmt, otherwise) + } + if err != nil { + return nil, err } if cond != nil && sel != nilSelector { From 29eed25df71d98e6a2c31f5018432e8c523986e8 Mon Sep 17 00:00:00 2001 From: urso Date: Wed, 5 Feb 2020 19:07:10 +0100 Subject: [PATCH 7/8] re-export symbols and add godoc --- libbeat/outputs/outil/select.go | 74 +++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index 6ad0ef515dd..d06ee4e3209 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -27,10 +27,14 @@ import ( "github.com/elastic/beats/libbeat/conditions" ) +// Selector is used to produce a string based on the contents of a Beats event. +// A selector supports multiple rules that need to be configured. type Selector struct { - sel selectorExpr + sel SelectorExpr } +// Settings configures how BuildSelectorFromConfig creates a Selector from +// a given configuration object. type Settings struct { // single selector key and default option keyword Key string @@ -45,18 +49,20 @@ type Settings struct { FailEmpty bool } -type selectorExpr interface { +// SelectorExpr represents an expression object that can be composed with other +// expressions in order to build a Selector. +type SelectorExpr interface { sel(evt *beat.Event) (string, error) } type emptySelector struct{} type listSelector struct { - selectors []selectorExpr + selectors []SelectorExpr } type condSelector struct { - s selectorExpr + s SelectorExpr cond conditions.Condition } @@ -70,21 +76,22 @@ type fmtSelector struct { } type mapSelector struct { - from selectorExpr + from SelectorExpr otherwise string to map[string]string } -var nilSelector selectorExpr = &emptySelector{} +var nilSelector SelectorExpr = &emptySelector{} -func MakeSelector(es ...selectorExpr) Selector { +// MakeSelector creates a selector from a set of selector expressions. +func MakeSelector(es ...SelectorExpr) Selector { switch len(es) { case 0: return Selector{nilSelector} case 1: return Selector{es[0]} default: - return Selector{concatSelectorExpr(es...)} + return Selector{ConcatSelectorExpr(es...)} } } @@ -96,10 +103,12 @@ func (s Selector) Select(evt *beat.Event) (string, error) { return s.sel.sel(evt) } +// IsEmpty checks if the selector is not configured and will always return an empty string. func (s Selector) IsEmpty() bool { return s.sel == nilSelector || s.sel == nil } +// IsConst checks if the selector will always return the same string. func (s Selector) IsConst() bool { if s.sel == nilSelector { return true @@ -109,11 +118,12 @@ func (s Selector) IsConst() bool { return ok } +// BuildSelectorFromConfig creates a selector from a configuration object. func BuildSelectorFromConfig( cfg *common.Config, settings Settings, ) (Selector, error) { - var sel []selectorExpr + var sel []SelectorExpr key := settings.Key multiKey := settings.MultiKey @@ -157,7 +167,7 @@ func BuildSelectorFromConfig( return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } - fmtsel, err := fmtSelectorExpr(fmtstr, "") + fmtsel, err := FmtSelectorExpr(fmtstr, "") if err != nil { return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } @@ -180,18 +190,22 @@ func BuildSelectorFromConfig( return MakeSelector(sel...), nil } -func emptySelectorExpr() selectorExpr { +// EmptySelectorExpr create a selector expression that returns an empty string. +func EmptySelectorExpr() SelectorExpr { return nilSelector } -func constSelectorExpr(s string) selectorExpr { +// ConstSelectorExpr creates a selector expression that always returns the configured string. +func ConstSelectorExpr(s string) SelectorExpr { if s == "" { - return emptySelectorExpr() + return EmptySelectorExpr() } return &constSelector{strings.ToLower(s)} } -func fmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (selectorExpr, error) { +// FmtSelectorExpr creates a selector expression using a format string. If the +// event can not be applied the default fallback constant string will be returned. +func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorExpr, error) { if fmt.IsConst() { str, err := fmt.Run(nil) if err != nil { @@ -200,28 +214,34 @@ func fmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (selectorEx if str == "" { str = fallback } - return constSelectorExpr(str), nil + return ConstSelectorExpr(str), nil } return &fmtSelector{*fmt, strings.ToLower(fallback)}, nil } -func concatSelectorExpr(s ...selectorExpr) selectorExpr { +// ConcatSelectorExpr combines multiple expressions that are run one after the other. +// The first expression that returns a string wins. +func ConcatSelectorExpr(s ...SelectorExpr) SelectorExpr { return &listSelector{s} } -func conditionalSelectorExpr( - s selectorExpr, +// ConditionalSelectorExpr executes the given expression only if the event +// matches the given condition. +func ConditionalSelectorExpr( + s SelectorExpr, cond conditions.Condition, -) selectorExpr { +) SelectorExpr { return &condSelector{s, cond} } -func lookupSelectorExpr( +// LookupSelectorExpr replaces the produced string with an table entry. +// If there is no entry in the table the default fallback string will be reported. +func LookupSelectorExpr( evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, -) (selectorExpr, error) { +) (SelectorExpr, error) { if evtfmt.IsConst() { str, err := evtfmt.Run(nil) if err != nil { @@ -232,7 +252,7 @@ func lookupSelectorExpr( if str == "" { str = fallback } - return constSelectorExpr(str), nil + return ConstSelectorExpr(str), nil } return &mapSelector{ @@ -250,7 +270,7 @@ func lowercaseTable(table map[string]string) map[string]string { return tmp } -func buildSingle(cfg *common.Config, key string) (selectorExpr, error) { +func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { // TODO: check for unknown fields // 1. extract required key-word handler @@ -310,18 +330,18 @@ func buildSingle(cfg *common.Config, key string) (selectorExpr, error) { } // 5. build selector from available fields - var sel selectorExpr + var sel SelectorExpr if len(mapping.Table) > 0 { - sel, err = lookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) + sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) } else { - sel, err = fmtSelectorExpr(evtfmt, otherwise) + sel, err = FmtSelectorExpr(evtfmt, otherwise) } if err != nil { return nil, err } if cond != nil && sel != nilSelector { - sel = conditionalSelectorExpr(sel, cond) + sel = ConditionalSelectorExpr(sel, cond) } return sel, nil From c9aa41a7f5d4403787854dd2c2433f8f3ba59077 Mon Sep 17 00:00:00 2001 From: urso Date: Thu, 6 Feb 2020 15:42:35 +0100 Subject: [PATCH 8/8] fix logstash integration test build --- libbeat/outputs/logstash/logstash_integration_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 8941984b922..6aca15337fe 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -92,7 +92,8 @@ func esConnect(t *testing.T, index string) *esConnection { host := getElasticsearchHost() indexFmt := fmtstr.MustCompileEvent(fmt.Sprintf("%s-%%{+yyyy.MM.dd}", index)) - indexSel := outil.MakeSelector(outil.FmtSelectorExpr(indexFmt, "")) + indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "") + indexSel := outil.MakeSelector(indexFmtExpr) index, _ = indexSel.Select(&beat.Event{ Timestamp: ts, })