From 6bc7e55aba0df842d0fe30109aa5a89b1877680d Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 29 May 2020 17:09:33 +0200 Subject: [PATCH] Use indexers and matchers in config when defaults are enabled (#18818) Before 7.7.0, indexers and matchers defined in `add_kubernetes_metadata` configuration were used even when defaults were not disabled. Revert to this behaviour and add tests to avoid changing it unexpectedly in the future. (cherry picked from commit 600998bd02773314ea0596e42b84105fb335e37b) Co-authored-by: Chris Mark --- CHANGELOG.next.asciidoc | 1 + .../add_kubernetes_metadata/kubernetes.go | 39 ++++--- .../kubernetes_test.go | 101 ++++++++++++++++++ 3 files changed, 126 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2f8aaa4cbe2..ede523a62df 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -117,6 +117,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Gives monitoring reporter hosts, if configured, total precedence over corresponding output hosts. {issue}17937[17937] {pull}17991[17991] - [Autodiscover] Check if runner is already running before starting again. {pull}18564[18564] - Fix `keystore add` hanging under Windows. {issue}18649[18649] {pull}18654[18654] +- Fix regression in `add_kubernetes_metadata`, so configured `indexers` and `matchers` are used if defaults are not disabled. {issue}18481[18481] {pull}18818[18818] *Auditbeat* diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index 9b7bc5653f8..94bc3739145 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -90,24 +90,12 @@ func isKubernetesAvailableWithRetry(client k8sclient.Interface) bool { // New constructs a new add_kubernetes_metadata processor. func New(cfg *common.Config) (processors.Processor, error) { - config := defaultKubernetesAnnotatorConfig() - log := logp.NewLogger(selector).With("libbeat.processor", "add_kubernetes_metadata") - - err := cfg.Unpack(&config) + config, err := newProcessorConfig(cfg, Indexing) if err != nil { - return nil, fmt.Errorf("fail to unpack the kubernetes configuration: %s", err) - } - - //Load default indexer configs - if config.DefaultIndexers.Enabled == true { - config.Indexers = Indexing.GetDefaultIndexerConfigs() - } - - //Load default matcher configs - if config.DefaultMatchers.Enabled == true { - config.Matchers = Indexing.GetDefaultMatcherConfigs() + return nil, err } + log := logp.NewLogger(selector).With("libbeat.processor", "add_kubernetes_metadata") processor := &kubernetesAnnotator{ log: log, cache: newCache(config.CleanupTimeout), @@ -121,6 +109,27 @@ func New(cfg *common.Config) (processors.Processor, error) { return processor, nil } +func newProcessorConfig(cfg *common.Config, register *Register) (kubeAnnotatorConfig, error) { + config := defaultKubernetesAnnotatorConfig() + + err := cfg.Unpack(&config) + if err != nil { + return config, fmt.Errorf("fail to unpack the kubernetes configuration: %s", err) + } + + //Load and append default indexer configs + if config.DefaultIndexers.Enabled { + config.Indexers = append(config.Indexers, register.GetDefaultIndexerConfigs()...) + } + + //Load and append default matcher configs + if config.DefaultMatchers.Enabled { + config.Matchers = append(config.Matchers, register.GetDefaultMatcherConfigs()...) + } + + return config, nil +} + func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *common.Config) { k.initOnce.Do(func() { client, err := kubernetes.GetKubernetesClient(config.KubeConfig) diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go b/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go index dec8f446fdc..a687fe6f8e3 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -128,3 +129,103 @@ func TestAnnotatorWithNoKubernetesAvailable(t *testing.T) { assert.Equal(t, intialEventMap, event.Fields) } + +// TestNewProcessorConfigDefaultIndexers validates the behaviour of default indexers and +// matchers settings +func TestNewProcessorConfigDefaultIndexers(t *testing.T) { + emptyRegister := NewRegister() + registerWithDefaults := NewRegister() + registerWithDefaults.AddDefaultIndexerConfig("ip_port", *common.NewConfig()) + registerWithDefaults.AddDefaultMatcherConfig("field_format", *common.MustNewConfigFrom(map[string]interface{}{ + "format": "%{[destination.ip]}:%{[destination.port]}", + })) + + configWithIndexersAndMatchers := common.MustNewConfigFrom(map[string]interface{}{ + "indexers": []map[string]interface{}{ + { + "container": map[string]interface{}{}, + }, + }, + "matchers": []map[string]interface{}{ + { + "fields": map[string]interface{}{ + "lookup_fields": []string{"container.id"}, + }, + }, + }, + }) + configOverrideDefaults := common.MustNewConfigFrom(map[string]interface{}{ + "default_indexers.enabled": "false", + "default_matchers.enabled": "false", + }) + require.NoError(t, configOverrideDefaults.Merge(configWithIndexersAndMatchers)) + + cases := map[string]struct { + register *Register + config *common.Config + expectedMatchers []string + expectedIndexers []string + }{ + "no matchers": { + register: emptyRegister, + config: common.NewConfig(), + }, + "one configured indexer and matcher": { + register: emptyRegister, + config: configWithIndexersAndMatchers, + expectedIndexers: []string{"container"}, + expectedMatchers: []string{"fields"}, + }, + "default indexers and matchers": { + register: registerWithDefaults, + config: common.NewConfig(), + expectedIndexers: []string{"ip_port"}, + expectedMatchers: []string{"field_format"}, + }, + "default indexers and matchers, don't use indexers": { + register: registerWithDefaults, + config: common.MustNewConfigFrom(map[string]interface{}{ + "default_indexers.enabled": "false", + }), + expectedMatchers: []string{"field_format"}, + }, + "default indexers and matchers, don't use matchers": { + register: registerWithDefaults, + config: common.MustNewConfigFrom(map[string]interface{}{ + "default_matchers.enabled": "false", + }), + expectedIndexers: []string{"ip_port"}, + }, + "one configured indexer and matcher and defaults, configured should come first": { + register: registerWithDefaults, + config: configWithIndexersAndMatchers, + expectedIndexers: []string{"container", "ip_port"}, + expectedMatchers: []string{"fields", "field_format"}, + }, + "override defaults": { + register: registerWithDefaults, + config: configOverrideDefaults, + expectedIndexers: []string{"container"}, + expectedMatchers: []string{"fields"}, + }, + } + + names := func(plugins PluginConfig) []string { + var ns []string + for _, plugin := range plugins { + for name := range plugin { + ns = append(ns, name) + } + } + return ns + } + + for title, c := range cases { + t.Run(title, func(t *testing.T) { + config, err := newProcessorConfig(c.config, c.register) + require.NoError(t, err) + assert.Equal(t, c.expectedMatchers, names(config.Matchers), "expected matchers") + assert.Equal(t, c.expectedIndexers, names(config.Indexers), "expected indexers") + }) + } +}