From 5fa9630f4fec8d699b4ce5c83a417873d341e6bf Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Thu, 11 Jun 2020 12:22:40 +0200 Subject: [PATCH] Cherry-pick #18854 to 7.x: Make selector string casing configurable (#19118) Make selector string casing configurable (#18854) Add support for configuring the string casing in the index/pipeline/key/topic 'Selector'. Elasticsearch pipeline and index names are required to be lower case only. When used with fields from events this was not always guaranteed, leading us to enforce lower case always (#16081. #6342). As the code is reused for Kafka topic selection, this unfortunately did lead to a Regression as some users expect strings to allow mixed case (#18640). With this PR Elasticsearch related resources (e.g. index or pipeline names) are set to lowercase only, while not touching the strings in other outputs. (cherry picked from commit 28f7aca29d91b86fb4159dae168b9051cac65593) --- CHANGELOG.next.asciidoc | 2 + libbeat/idxmgmt/std.go | 8 +- libbeat/idxmgmt/std_test.go | 44 ++++ libbeat/outputs/elasticsearch/client.go | 3 +- .../elasticsearch/client_proxy_test.go | 2 +- libbeat/outputs/elasticsearch/client_test.go | 2 +- .../outputs/elasticsearch/elasticsearch.go | 17 +- .../elasticsearch/elasticsearch_test.go | 58 +++++ libbeat/outputs/kafka/config_test.go | 62 +++++ libbeat/outputs/kafka/kafka.go | 17 +- .../logstash/logstash_integration_test.go | 2 +- libbeat/outputs/outil/select.go | 51 ++-- libbeat/outputs/outil/select_test.go | 221 +++++++++++------- libbeat/outputs/outil/settings.go | 96 ++++++++ libbeat/outputs/redis/redis.go | 17 +- libbeat/outputs/redis/redis_test.go | 61 +++++ 16 files changed, 526 insertions(+), 137 deletions(-) create mode 100644 libbeat/outputs/outil/settings.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 388e686b8be..dc32a730211 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -143,6 +143,8 @@ field. You can revert this change by configuring tags for the module and omittin - Fix the `translate_sid` processor's handling of unconfigured target fields. {issue}18990[18990] {pull}18991[18991] - Fixed a service restart failure under Windows. {issue}18914[18914] {pull}18916[18916] - The `monitoring.elasticsearch.api_key` value is correctly base64-encoded before being sent to the monitoring Elasticsearch cluster. {issue}18939[18939] {pull}18945[18945] +- Fix kafka topic setting not allowing upper case characters. {pull}18854[18854] {issue}18640[18640] +- Fix redis key setting not allowing upper case characters. {pull}18854[18854] {issue}18640[18640] *Auditbeat* diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index 06a56646807..9aab5487301 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -20,6 +20,7 @@ package idxmgmt import ( "errors" "fmt" + "strings" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/beat/events" @@ -198,6 +199,7 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, MultiKey: "indices", EnableSingleOnly: true, FailEmpty: mode != ilm.ModeEnabled, + Case: outil.SelectorLowerCase, } indexSel, err := outil.BuildSelectorFromConfig(selCfg, buildSettings) @@ -354,13 +356,13 @@ func getEventCustomIndex(evt *beat.Event, beatInfo beat.Info) string { } if alias, err := events.GetMetaStringValue(*evt, events.FieldMetaAlias); err == nil { - return alias + return strings.ToLower(alias) } if idx, err := events.GetMetaStringValue(*evt, events.FieldMetaIndex); err == nil { ts := evt.Timestamp.UTC() return fmt.Sprintf("%s-%d.%02d.%02d", - idx, ts.Year(), ts.Month(), ts.Day()) + strings.ToLower(idx), ts.Year(), ts.Month(), ts.Day()) } // This is functionally identical to Meta["alias"], returning the overriding @@ -368,7 +370,7 @@ func getEventCustomIndex(evt *beat.Event, beatInfo beat.Info) string { // to send the index for particular inputs to formatted string templates, // which are then expanded by a processor to the "raw_index" field. if idx, err := events.GetMetaStringValue(*evt, events.FieldMetaRawIndex); err == nil { - return idx + return strings.ToLower(idx) } return "" diff --git a/libbeat/idxmgmt/std_test.go b/libbeat/idxmgmt/std_test.go index ea23a53fd84..3e934d78d58 100644 --- a/libbeat/idxmgmt/std_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -139,6 +139,11 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, want: stable("test-9.9.9"), }, + "without ilm must be lowercase": { + ilmCalls: noILM, + cfg: map[string]interface{}{"index": "TeSt-%{[agent.version]}"}, + want: stable("test-9.9.9"), + }, "event alias without ilm": { ilmCalls: noILM, cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -147,6 +152,14 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { "alias": "test", }, }, + "event alias without ilm must be lowercae": { + ilmCalls: noILM, + cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, + want: stable("test"), + meta: common.MapStr{ + "alias": "Test", + }, + }, "event index without ilm": { ilmCalls: noILM, cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -155,11 +168,24 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { "index": "test", }, }, + "event index without ilm must be lowercase": { + ilmCalls: noILM, + cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, + want: dateIdx("test"), + meta: common.MapStr{ + "index": "Test", + }, + }, "with ilm": { ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), cfg: map[string]interface{}{"index": "wrong-%{[agent.version]}"}, want: stable("test-9.9.9"), }, + "with ilm must be lowercase": { + ilmCalls: ilmTemplateSettings("Test-9.9.9", "Test-9.9.9"), + cfg: map[string]interface{}{"index": "wrong-%{[agent.version]}"}, + want: stable("test-9.9.9"), + }, "event alias wit ilm": { ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -168,6 +194,14 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { "alias": "event-alias", }, }, + "event alias wit ilm must be lowercase": { + ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), + cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, + want: stable("event-alias"), + meta: common.MapStr{ + "alias": "Event-alias", + }, + }, "event index with ilm": { ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), cfg: map[string]interface{}{"index": "test-%{[agent.version]}"}, @@ -186,6 +220,16 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { }, want: stable("myindex"), }, + "use indices settings must be lowercase": { + ilmCalls: ilmTemplateSettings("test-9.9.9", "test-9.9.9"), + cfg: map[string]interface{}{ + "index": "test-%{[agent.version]}", + "indices": []map[string]interface{}{ + {"index": "MyIndex"}, + }, + }, + want: stable("myindex"), + }, } for name, test := range cases { t.Run(name, func(t *testing.T) { diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index c9df4c1bab4..3afa7084057 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "strings" "time" "go.elastic.co/apm" @@ -352,7 +353,7 @@ func getPipeline(event *beat.Event, pipelineSel *outil.Selector) (string, error) return "", errors.New("pipeline metadata is no string") } - return pipeline, nil + return strings.ToLower(pipeline), nil } if pipelineSel != nil { diff --git a/libbeat/outputs/elasticsearch/client_proxy_test.go b/libbeat/outputs/elasticsearch/client_proxy_test.go index b6751860e0a..1e368d234ea 100644 --- a/libbeat/outputs/elasticsearch/client_proxy_test.go +++ b/libbeat/outputs/elasticsearch/client_proxy_test.go @@ -190,7 +190,7 @@ func doClientPing(t *testing.T) { Headers: map[string]string{headerTestField: headerTestValue}, ProxyDisable: proxyDisable != "", }, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), + Index: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)), } if proxy != "" { proxyURL, err := url.Parse(proxy) diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index db152bf9045..bd28fe5850b 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -228,7 +228,7 @@ func TestClientWithHeaders(t *testing.T) { "X-Test": "testing value", }, }, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test")), + Index: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)), }, nil) assert.NoError(t, err) diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 512b74895ea..bf1f9bd378e 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -133,12 +133,7 @@ func buildSelectors( return index, pipeline, err } - pipelineSel, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ - Key: "pipeline", - MultiKey: "pipelines", - EnableSingleOnly: true, - FailEmpty: false, - }) + pipelineSel, err := buildPipelineSelector(cfg) if err != nil { return index, pipeline, err } @@ -149,3 +144,13 @@ func buildSelectors( return index, pipeline, err } + +func buildPipelineSelector(cfg *common.Config) (outil.Selector, error) { + return outil.BuildSelectorFromConfig(cfg, outil.Settings{ + Key: "pipeline", + MultiKey: "pipelines", + EnableSingleOnly: true, + FailEmpty: false, + Case: outil.SelectorLowerCase, + }) +} diff --git a/libbeat/outputs/elasticsearch/elasticsearch_test.go b/libbeat/outputs/elasticsearch/elasticsearch_test.go index 60268b59602..df757d570dd 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch_test.go +++ b/libbeat/outputs/elasticsearch/elasticsearch_test.go @@ -21,6 +21,8 @@ import ( "fmt" "testing" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" ) @@ -73,3 +75,59 @@ func TestGlobalConnectCallbacksManagement(t *testing.T) { t.Fatalf("third callback cannot be retrieved") } } + +func TestPipelineSelection(t *testing.T) { + cases := map[string]struct { + cfg map[string]interface{} + event beat.Event + want string + }{ + "no pipline configured": {}, + "pipeline configured": { + cfg: map[string]interface{}{"pipeline": "test"}, + want: "test", + }, + "pipeline must be lowercase": { + cfg: map[string]interface{}{"pipeline": "Test"}, + want: "test", + }, + "pipeline via event meta": { + event: beat.Event{Meta: common.MapStr{"pipeline": "test"}}, + want: "test", + }, + "pipeline via event meta must be lowercase": { + event: beat.Event{Meta: common.MapStr{"pipeline": "Test"}}, + want: "test", + }, + "pipelines setting": { + cfg: map[string]interface{}{ + "pipelines": []map[string]interface{}{{"pipeline": "test"}}, + }, + want: "test", + }, + "pipelines setting must be lowercase": { + cfg: map[string]interface{}{ + "pipelines": []map[string]interface{}{{"pipeline": "Test"}}, + }, + want: "test", + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + selector, err := buildPipelineSelector(common.MustNewConfigFrom(test.cfg)) + if err != nil { + t.Fatalf("Failed to parse configuration: %v", err) + } + + got, err := getPipeline(&test.event, &selector) + if err != nil { + t.Fatalf("Failed to create pipeline name: %v", err) + } + + if test.want != got { + t.Errorf("Pipeline name missmatch (want: %v, got: %v)", test.want, got) + } + }) + } +} diff --git a/libbeat/outputs/kafka/config_test.go b/libbeat/outputs/kafka/config_test.go index 3775cafd47b..816cb8f03e8 100644 --- a/libbeat/outputs/kafka/config_test.go +++ b/libbeat/outputs/kafka/config_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/internal/testutil" "github.com/elastic/beats/v7/libbeat/logp" @@ -131,3 +132,64 @@ func TestBackoffFunc(t *testing.T) { }) } } + +func TestTopicSelection(t *testing.T) { + cases := map[string]struct { + cfg map[string]interface{} + event beat.Event + want string + }{ + "topic configured": { + cfg: map[string]interface{}{"topic": "test"}, + want: "test", + }, + "topic must keep case": { + cfg: map[string]interface{}{"topic": "Test"}, + want: "Test", + }, + "topics setting": { + cfg: map[string]interface{}{ + "topics": []map[string]interface{}{{"topic": "test"}}, + }, + want: "test", + }, + "topics setting must keep case": { + cfg: map[string]interface{}{ + "topics": []map[string]interface{}{{"topic": "Test"}}, + }, + want: "Test", + }, + "use event field": { + cfg: map[string]interface{}{"topic": "test-%{[field]}"}, + event: beat.Event{ + Fields: common.MapStr{"field": "from-event"}, + }, + want: "test-from-event", + }, + "use event field must keep case": { + cfg: map[string]interface{}{"topic": "Test-%{[field]}"}, + event: beat.Event{ + Fields: common.MapStr{"field": "From-Event"}, + }, + want: "Test-From-Event", + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + selector, err := buildTopicSelector(common.MustNewConfigFrom(test.cfg)) + if err != nil { + t.Fatalf("Failed to parse configuration: %v", err) + } + + got, err := selector.Select(&test.event) + if err != nil { + t.Fatalf("Failed to create topic name: %v", err) + } + + if test.want != got { + t.Errorf("Pipeline name missmatch (want: %v, got: %v)", test.want, got) + } + }) + } +} diff --git a/libbeat/outputs/kafka/kafka.go b/libbeat/outputs/kafka/kafka.go index b8c6c4dcfff..9be3970b1c4 100644 --- a/libbeat/outputs/kafka/kafka.go +++ b/libbeat/outputs/kafka/kafka.go @@ -66,12 +66,7 @@ func makeKafka( return outputs.Fail(err) } - topic, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ - Key: "topic", - MultiKey: "topics", - EnableSingleOnly: true, - FailEmpty: true, - }) + topic, err := buildTopicSelector(cfg) if err != nil { return outputs.Fail(err) } @@ -102,3 +97,13 @@ func makeKafka( } return outputs.Success(config.BulkMaxSize, retry, client) } + +func buildTopicSelector(cfg *common.Config) (outil.Selector, error) { + return outil.BuildSelectorFromConfig(cfg, outil.Settings{ + Key: "topic", + MultiKey: "topics", + EnableSingleOnly: true, + FailEmpty: true, + Case: outil.SelectorKeepCase, + }) +} diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go index 0c744e470cb..872a7f0a01d 100644 --- a/libbeat/outputs/logstash/logstash_integration_test.go +++ b/libbeat/outputs/logstash/logstash_integration_test.go @@ -94,7 +94,7 @@ func esConnect(t *testing.T, index string) *esConnection { host := getElasticsearchHost() indexFmt := fmtstr.MustCompileEvent(fmt.Sprintf("%s-%%{+yyyy.MM.dd}", index)) - indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "") + indexFmtExpr, _ := outil.FmtSelectorExpr(indexFmt, "", outil.SelectorLowerCase) indexSel := outil.MakeSelector(indexFmtExpr) index, _ = indexSel.Select(&beat.Event{ Timestamp: ts, diff --git a/libbeat/outputs/outil/select.go b/libbeat/outputs/outil/select.go index 6ff629c88e7..1615a3bdb11 100644 --- a/libbeat/outputs/outil/select.go +++ b/libbeat/outputs/outil/select.go @@ -19,7 +19,6 @@ package outil import ( "fmt" - "strings" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -33,22 +32,6 @@ type Selector struct { sel SelectorExpr } -// Settings configures how BuildSelectorFromConfig creates a Selector from -// a given configuration object. -type Settings struct { - // single selector key and default option keyword - Key string - - // multi-selector key in config - MultiKey string - - // if enabled a selector `key` in config will be generated, if `key` is present - EnableSingleOnly bool - - // Fail building selector if `key` and `multiKey` are missing - FailEmpty bool -} - // SelectorExpr represents an expression object that can be composed with other // expressions in order to build a Selector. type SelectorExpr interface { @@ -73,6 +56,7 @@ type constSelector struct { type fmtSelector struct { f fmtstr.EventFormatString otherwise string + selCase SelectorCase } type mapSelector struct { @@ -142,7 +126,7 @@ func BuildSelectorFromConfig( } for _, config := range table { - action, err := buildSingle(config, key) + action, err := buildSingle(config, key, settings.Case) if err != nil { return Selector{}, err } @@ -167,7 +151,7 @@ func BuildSelectorFromConfig( return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } - fmtsel, err := FmtSelectorExpr(fmtstr, "") + fmtsel, err := FmtSelectorExpr(fmtstr, "", settings.Case) if err != nil { return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key)) } @@ -196,16 +180,16 @@ func EmptySelectorExpr() SelectorExpr { } // ConstSelectorExpr creates a selector expression that always returns the configured string. -func ConstSelectorExpr(s string) SelectorExpr { +func ConstSelectorExpr(s string, selCase SelectorCase) SelectorExpr { if s == "" { return EmptySelectorExpr() } - return &constSelector{strings.ToLower(s)} + return &constSelector{selCase.apply(s)} } // FmtSelectorExpr creates a selector expression using a format string. If the // event can not be applied the default fallback constant string will be returned. -func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorExpr, error) { +func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string, selCase SelectorCase) (SelectorExpr, error) { if fmt.IsConst() { str, err := fmt.Run(nil) if err != nil { @@ -214,10 +198,10 @@ func FmtSelectorExpr(fmt *fmtstr.EventFormatString, fallback string) (SelectorEx if str == "" { str = fallback } - return ConstSelectorExpr(str), nil + return ConstSelectorExpr(str, selCase), nil } - return &fmtSelector{*fmt, strings.ToLower(fallback)}, nil + return &fmtSelector{*fmt, selCase.apply(fallback), selCase}, nil } // ConcatSelectorExpr combines multiple expressions that are run one after the other. @@ -241,6 +225,7 @@ func LookupSelectorExpr( evtfmt *fmtstr.EventFormatString, table map[string]string, fallback string, + selCase SelectorCase, ) (SelectorExpr, error) { if evtfmt.IsConst() { str, err := evtfmt.Run(nil) @@ -248,11 +233,11 @@ func LookupSelectorExpr( return nil, err } - str = table[strings.ToLower(str)] + str = table[selCase.apply(str)] if str == "" { str = fallback } - return ConstSelectorExpr(str), nil + return ConstSelectorExpr(str, selCase), nil } return &mapSelector{ @@ -262,15 +247,15 @@ func LookupSelectorExpr( }, nil } -func lowercaseTable(table map[string]string) map[string]string { +func copyTable(selCase SelectorCase, table map[string]string) map[string]string { tmp := make(map[string]string, len(table)) for k, v := range table { - tmp[strings.ToLower(k)] = strings.ToLower(v) + tmp[selCase.apply(k)] = selCase.apply(v) } return tmp } -func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { +func buildSingle(cfg *common.Config, key string, selCase SelectorCase) (SelectorExpr, error) { // TODO: check for unknown fields // 1. extract required key-word handler @@ -295,7 +280,7 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { if err != nil { return nil, err } - otherwise = strings.ToLower(tmp) + otherwise = selCase.apply(tmp) } // 3. extract optional `mapping` @@ -332,9 +317,9 @@ func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) { // 5. build selector from available fields var sel SelectorExpr if len(mapping.Table) > 0 { - sel, err = LookupSelectorExpr(evtfmt, lowercaseTable(mapping.Table), otherwise) + sel, err = LookupSelectorExpr(evtfmt, copyTable(selCase, mapping.Table), otherwise, selCase) } else { - sel, err = FmtSelectorExpr(evtfmt, otherwise) + sel, err = FmtSelectorExpr(evtfmt, otherwise, selCase) } if err != nil { return nil, err @@ -388,7 +373,7 @@ func (s *fmtSelector) sel(evt *beat.Event) (string, error) { if n == "" { return s.otherwise, nil } - return strings.ToLower(n), nil + return s.selCase.apply(n), nil } func (s *mapSelector) sel(evt *beat.Event) (string, error) { diff --git a/libbeat/outputs/outil/select_test.go b/libbeat/outputs/outil/select_test.go index e16cb602a96..49ea63bbd4b 100644 --- a/libbeat/outputs/outil/select_test.go +++ b/libbeat/outputs/outil/select_test.go @@ -31,181 +31,239 @@ import ( type node map[string]interface{} func TestSelector(t *testing.T) { + useLowerCase := func(s Settings) Settings { + return s.WithSelectorCase(SelectorLowerCase) + } + tests := map[string]struct { config string event common.MapStr - expected string + want string + settings func(Settings) Settings }{ "constant key": { - `key: value`, - common.MapStr{}, - "value", + config: `key: value`, + event: common.MapStr{}, + want: "value", }, "lowercase constant key": { - `key: VaLuE`, - common.MapStr{}, - "value", + config: `key: VaLuE`, + event: common.MapStr{}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase constant key by default": { + config: `key: VaLuE`, + event: common.MapStr{}, + want: "VaLuE", }, "format string key": { - `key: '%{[key]}'`, - common.MapStr{"key": "value"}, - "value", + config: `key: '%{[key]}'`, + event: common.MapStr{"key": "value"}, + want: "value", }, "lowercase format string key": { - `key: '%{[key]}'`, - common.MapStr{"key": "VaLuE"}, - "value", + config: `key: '%{[key]}'`, + event: common.MapStr{"key": "VaLuE"}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase format string by default": { + config: `key: '%{[key]}'`, + event: common.MapStr{"key": "VaLuE"}, + want: "VaLuE", }, "key with empty keys": { - `{key: value, keys: }`, - common.MapStr{}, - "value", + config: `{key: value, keys: }`, + event: common.MapStr{}, + want: "value", }, "lowercase key with empty keys": { - `{key: vAlUe, keys: }`, - common.MapStr{}, - "value", + config: `{key: vAlUe, keys: }`, + event: common.MapStr{}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase key with empty keys by default": { + config: `{key: vAlUe, keys: }`, + event: common.MapStr{}, + want: "vAlUe", }, "constant in multi key": { - `keys: [key: 'value']`, - common.MapStr{}, - "value", + config: `keys: [key: 'value']`, + event: common.MapStr{}, + want: "value", }, "format string in multi key": { - `keys: [key: '%{[key]}']`, - common.MapStr{"key": "value"}, - "value", + config: `keys: [key: '%{[key]}']`, + event: common.MapStr{"key": "value"}, + want: "value", }, "missing format string key with default in rule": { - `keys: + config: `keys: - key: '%{[key]}' default: value`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "lowercase missing format string key with default in rule": { - `keys: + config: `keys: + - key: '%{[key]}' + default: vAlUe`, + event: common.MapStr{}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase missing format string key with default in rule": { + config: `keys: - key: '%{[key]}' default: vAlUe`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "vAlUe", }, "empty format string key with default in rule": { - `keys: + config: `keys: - key: '%{[key]}' default: value`, - common.MapStr{"key": ""}, - "value", + event: common.MapStr{"key": ""}, + want: "value", }, "lowercase empty format string key with default in rule": { - `keys: + config: `keys: + - key: '%{[key]}' + default: vAluE`, + event: common.MapStr{"key": ""}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase empty format string key with default in rule": { + config: `keys: - key: '%{[key]}' default: vAluE`, - common.MapStr{"key": ""}, - "value", + event: common.MapStr{"key": ""}, + want: "vAluE", }, "missing format string key with constant in next rule": { - `keys: + config: `keys: - key: '%{[key]}' - key: value`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "missing format string key with constant in top-level rule": { - `{ key: value, keys: [key: '%{[key]}']}`, - common.MapStr{}, - "value", + config: `{ key: value, keys: [key: '%{[key]}']}`, + event: common.MapStr{}, + want: "value", }, "apply mapping": { - `keys: + config: `keys: - key: '%{[key]}' mappings: v: value`, - common.MapStr{"key": "v"}, - "value", + event: common.MapStr{"key": "v"}, + want: "value", }, "lowercase applied mapping": { - `keys: + config: `keys: + - key: '%{[key]}' + mappings: + v: vAlUe`, + event: common.MapStr{"key": "v"}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase applied mapping": { + config: `keys: - key: '%{[key]}' mappings: v: vAlUe`, - common.MapStr{"key": "v"}, - "value", + event: common.MapStr{"key": "v"}, + want: "vAlUe", }, "apply mapping with default on empty key": { - `keys: + config: `keys: - key: '%{[key]}' default: value mappings: v: 'v'`, - common.MapStr{"key": ""}, - "value", + event: common.MapStr{"key": ""}, + want: "value", }, "lowercase apply mapping with default on empty key": { - `keys: + config: `keys: + - key: '%{[key]}' + default: vAluE + mappings: + v: 'v'`, + event: common.MapStr{"key": ""}, + want: "value", + settings: useLowerCase, + }, + "do not lowercase apply mapping with default on empty key": { + config: `keys: - key: '%{[key]}' default: vAluE mappings: v: 'v'`, - common.MapStr{"key": ""}, - "value", + event: common.MapStr{"key": ""}, + want: "vAluE", }, "apply mapping with default on empty lookup": { - `keys: + config: `keys: - key: '%{[key]}' default: value mappings: v: ''`, - common.MapStr{"key": "v"}, - "value", + event: common.MapStr{"key": "v"}, + want: "value", }, "apply mapping without match": { - `keys: + config: `keys: - key: '%{[key]}' mappings: v: '' - key: value`, - common.MapStr{"key": "x"}, - "value", + event: common.MapStr{"key": "x"}, + want: "value", }, "mapping with constant key": { - `keys: + config: `keys: - key: k mappings: k: value`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "mapping with missing constant key": { - `keys: + config: `keys: - key: unknown mappings: {k: wrong} - key: value`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "mapping with missing constant key, but default": { - `keys: + config: `keys: - key: unknown default: value mappings: {k: wrong}`, - common.MapStr{}, - "value", + event: common.MapStr{}, + want: "value", }, "matching condition": { - `keys: + config: `keys: - key: value when.equals.test: test`, - common.MapStr{"test": "test"}, - "value", + event: common.MapStr{"test": "test"}, + want: "value", }, "failing condition": { - `keys: + config: `keys: - key: wrong when.equals.test: test - key: value`, - common.MapStr{"test": "x"}, - "value", + event: common.MapStr{"test": "x"}, + want: "value", }, } @@ -217,12 +275,17 @@ func TestSelector(t *testing.T) { t.Fatalf("YAML parse error: %v\n%v", err, yaml) } - sel, err := BuildSelectorFromConfig(cfg, Settings{ + settings := Settings{ Key: "key", MultiKey: "keys", EnableSingleOnly: true, FailEmpty: true, - }) + } + if test.settings != nil { + settings = test.settings(settings) + } + + sel, err := BuildSelectorFromConfig(cfg, settings) if err != nil { t.Fatal(err) } @@ -236,7 +299,7 @@ func TestSelector(t *testing.T) { t.Fatal(err) } - assert.Equal(t, test.expected, actual) + assert.Equal(t, test.want, actual) }) } } diff --git a/libbeat/outputs/outil/settings.go b/libbeat/outputs/outil/settings.go new file mode 100644 index 00000000000..380f75d23e6 --- /dev/null +++ b/libbeat/outputs/outil/settings.go @@ -0,0 +1,96 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package outil + +import "strings" + +// Settings configures how BuildSelectorFromConfig creates a Selector from +// a given configuration object. +type Settings struct { + // single selector key and default option keyword + Key string + + // multi-selector key in config + MultiKey string + + // if enabled a selector `key` in config will be generated, if `key` is present + EnableSingleOnly bool + + // Fail building selector if `key` and `multiKey` are missing + FailEmpty bool + + // Case configures the case-sensitivity of generated strings. + Case SelectorCase +} + +// SelectorCase is used to configure a Selector output string casing. +// Use SelectorLowerCase or SelectorUpperCase to enforce the Selector to +// always generate lower case or upper case strings. +type SelectorCase uint8 + +const ( + // SelectorKeepCase instructs the Selector to not modify the string output. + SelectorKeepCase SelectorCase = iota + + // SelectorLowerCase instructs the Selector to always transform the string output to lower case only. + SelectorLowerCase + + // SelectorUpperCase instructs the Selector to always transform the string output to upper case only. + SelectorUpperCase +) + +// WithKey returns a new Settings struct with updated `Key` setting. +func (s Settings) WithKey(key string) Settings { + s.Key = key + return s +} + +// WithMultiKey returns a new Settings struct with updated `MultiKey` setting. +func (s Settings) WithMultiKey(key string) Settings { + s.MultiKey = key + return s +} + +// WithEnableSingleOnly returns a new Settings struct with updated `EnableSingleOnly` setting. +func (s Settings) WithEnableSingleOnly(b bool) Settings { + s.EnableSingleOnly = b + return s +} + +// WithFailEmpty returns a new Settings struct with updated `FailEmpty` setting. +func (s Settings) WithFailEmpty(b bool) Settings { + s.FailEmpty = b + return s +} + +// WithSelectorCase returns a new Settings struct with updated `Case` setting. +func (s Settings) WithSelectorCase(c SelectorCase) Settings { + s.Case = c + return s +} + +func (selCase SelectorCase) apply(in string) string { + switch selCase { + case SelectorLowerCase: + return strings.ToLower(in) + case SelectorUpperCase: + return strings.ToUpper(in) + default: + return in + } +} diff --git a/libbeat/outputs/redis/redis.go b/libbeat/outputs/redis/redis.go index 7ef312016e3..910b69d9f58 100644 --- a/libbeat/outputs/redis/redis.go +++ b/libbeat/outputs/redis/redis.go @@ -92,12 +92,7 @@ func makeRedis( return outputs.Fail(errors.New("Bad Redis data type")) } - key, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ - Key: "key", - MultiKey: "keys", - EnableSingleOnly: true, - FailEmpty: true, - }) + key, err := buildKeySelector(cfg) if err != nil { return outputs.Fail(err) } @@ -174,3 +169,13 @@ func makeRedis( return outputs.SuccessNet(config.LoadBalance, config.BulkMaxSize, config.MaxRetries, clients) } + +func buildKeySelector(cfg *common.Config) (outil.Selector, error) { + return outil.BuildSelectorFromConfig(cfg, outil.Settings{ + Key: "key", + MultiKey: "keys", + EnableSingleOnly: true, + FailEmpty: true, + Case: outil.SelectorKeepCase, + }) +} diff --git a/libbeat/outputs/redis/redis_test.go b/libbeat/outputs/redis/redis_test.go index 5ca91d3fef0..deffb1d8de5 100644 --- a/libbeat/outputs/redis/redis_test.go +++ b/libbeat/outputs/redis/redis_test.go @@ -118,3 +118,64 @@ func TestMakeRedis(t *testing.T) { }) } } + +func TestKeySelection(t *testing.T) { + cases := map[string]struct { + cfg map[string]interface{} + event beat.Event + want string + }{ + "key configured": { + cfg: map[string]interface{}{"key": "test"}, + want: "test", + }, + "key must keep case": { + cfg: map[string]interface{}{"key": "Test"}, + want: "Test", + }, + "key setting": { + cfg: map[string]interface{}{ + "keys": []map[string]interface{}{{"key": "test"}}, + }, + want: "test", + }, + "keys setting must keep case": { + cfg: map[string]interface{}{ + "keys": []map[string]interface{}{{"key": "Test"}}, + }, + want: "Test", + }, + "use event field": { + cfg: map[string]interface{}{"key": "test-%{[field]}"}, + event: beat.Event{ + Fields: common.MapStr{"field": "from-event"}, + }, + want: "test-from-event", + }, + "use event field must keep case": { + cfg: map[string]interface{}{"key": "Test-%{[field]}"}, + event: beat.Event{ + Fields: common.MapStr{"field": "From-Event"}, + }, + want: "Test-From-Event", + }, + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + selector, err := buildKeySelector(common.MustNewConfigFrom(test.cfg)) + if err != nil { + t.Fatalf("Failed to parse configuration: %v", err) + } + + got, err := selector.Select(&test.event) + if err != nil { + t.Fatalf("Failed to create key name: %v", err) + } + + if test.want != got { + t.Errorf("Pipeline name missmatch (want: %v, got: %v)", test.want, got) + } + }) + } +}