From 80af4ce6ef652a929bfbb1ccd0e28c1a303a48dd Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 25 Feb 2020 14:43:52 +0200 Subject: [PATCH 1/2] Fix k8s pod annotations tier in metadata Signed-off-by: chrismark --- libbeat/common/kubernetes/metadata/pod.go | 11 +++- .../common/kubernetes/metadata/pod_test.go | 61 +++++++++++++++---- .../add_kubernetes_metadata/indexers_test.go | 4 +- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go index 7385d2abdf5..018b710e4b4 100644 --- a/libbeat/common/kubernetes/metadata/pod.go +++ b/libbeat/common/kubernetes/metadata/pod.go @@ -50,7 +50,7 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map out := p.resource.Generate("pod", obj, opts...) // TODO: remove this call when moving to 8.0 - out = p.exportPodLabels(out) + out = p.exportPodLabelsAndAnnotations(out) if p.node != nil { meta := p.node.GenerateFromName(po.Spec.NodeName) @@ -92,7 +92,7 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr return nil } -func (p *pod) exportPodLabels(in common.MapStr) common.MapStr { +func (p *pod) exportPodLabelsAndAnnotations(in common.MapStr) common.MapStr { labels, err := in.GetValue("pod.labels") if err != nil { return in @@ -100,5 +100,12 @@ func (p *pod) exportPodLabels(in common.MapStr) common.MapStr { in.Put("labels", labels) in.Delete("pod.labels") + annotations, err := in.GetValue("pod.annotations") + if err != nil { + return in + } + in.Put("annotations", annotations) + in.Delete("pod.annotations") + return in } diff --git a/libbeat/common/kubernetes/metadata/pod_test.go b/libbeat/common/kubernetes/metadata/pod_test.go index c3838ad2c32..a5c3f2658a4 100644 --- a/libbeat/common/kubernetes/metadata/pod_test.go +++ b/libbeat/common/kubernetes/metadata/pod_test.go @@ -53,7 +53,9 @@ func TestPod_Generate(t *testing.T) { Labels: map[string]string{ "foo": "bar", }, - Annotations: map[string]string{}, + Annotations: map[string]string{ + "app": "production", + }, }, TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -71,6 +73,9 @@ func TestPod_Generate(t *testing.T) { "labels": common.MapStr{ "foo": "bar", }, + "annotations": common.MapStr{ + "app": "production", + }, "namespace": "default", "node": common.MapStr{ "name": "testnode", @@ -87,7 +92,9 @@ func TestPod_Generate(t *testing.T) { Labels: map[string]string{ "foo": "bar", }, - Annotations: map[string]string{}, + Annotations: map[string]string{ + "app": "production", + }, OwnerReferences: []metav1.OwnerReference{ { APIVersion: "apps", @@ -121,12 +128,19 @@ func TestPod_Generate(t *testing.T) { "labels": common.MapStr{ "foo": "bar", }, + "annotations": common.MapStr{ + "app": "production", + }, }, }, } - cfg := common.NewConfig() - metagen := NewPodMetadataGenerator(cfg, nil, nil, nil) + config, err := common.NewConfigFrom(map[string]interface{}{ + "include_annotations": []string{"app"}, + }) + assert.Nil(t, err) + + metagen := NewPodMetadataGenerator(config, nil, nil, nil) for _, test := range tests { t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.output, metagen.Generate(test.input)) @@ -154,7 +168,9 @@ func TestPod_GenerateFromName(t *testing.T) { Labels: map[string]string{ "foo": "bar", }, - Annotations: map[string]string{}, + Annotations: map[string]string{ + "app": "production", + }, }, TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -176,6 +192,9 @@ func TestPod_GenerateFromName(t *testing.T) { "labels": common.MapStr{ "foo": "bar", }, + "annotations": common.MapStr{ + "app": "production", + }, }, }, { @@ -188,7 +207,9 @@ func TestPod_GenerateFromName(t *testing.T) { Labels: map[string]string{ "foo": "bar", }, - Annotations: map[string]string{}, + Annotations: map[string]string{ + "app": "production", + }, OwnerReferences: []metav1.OwnerReference{ { APIVersion: "apps", @@ -222,15 +243,21 @@ func TestPod_GenerateFromName(t *testing.T) { "labels": common.MapStr{ "foo": "bar", }, + "annotations": common.MapStr{ + "app": "production", + }, }, }, } for _, test := range tests { - cfg := common.NewConfig() + config, err := common.NewConfigFrom(map[string]interface{}{ + "include_annotations": []string{"app"}, + }) + assert.Nil(t, err) pods := cache.NewStore(cache.MetaNamespaceKeyFunc) pods.Add(test.input) - metagen := NewPodMetadataGenerator(cfg, pods, nil, nil) + metagen := NewPodMetadataGenerator(config, pods, nil, nil) accessor, err := meta.Accessor(test.input) require.Nil(t, err) @@ -262,7 +289,9 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) { Labels: map[string]string{ "foo": "bar", }, - Annotations: map[string]string{}, + Annotations: map[string]string{ + "app": "production", + }, }, TypeMeta: metav1.TypeMeta{ Kind: "Pod", @@ -320,24 +349,30 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) { "labels": common.MapStr{ "foo": "bar", }, + "annotations": common.MapStr{ + "app": "production", + }, }, }, } for _, test := range tests { - cfg := common.NewConfig() + config, err := common.NewConfigFrom(map[string]interface{}{ + "include_annotations": []string{"app"}, + }) + assert.Nil(t, err) pods := cache.NewStore(cache.MetaNamespaceKeyFunc) pods.Add(test.input) nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) nodes.Add(test.node) - nodeMeta := NewNodeMetadataGenerator(cfg, nodes) + nodeMeta := NewNodeMetadataGenerator(config, nodes) namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) namespaces.Add(test.namespace) - nsMeta := NewNamespaceMetadataGenerator(cfg, namespaces) + nsMeta := NewNamespaceMetadataGenerator(config, namespaces) - metagen := NewPodMetadataGenerator(cfg, pods, nodeMeta, nsMeta) + metagen := NewPodMetadataGenerator(config, pods, nodeMeta, nsMeta) t.Run(test.name, func(t *testing.T) { assert.Equal(t, test.output, metagen.Generate(test.input)) }) diff --git a/libbeat/processors/add_kubernetes_metadata/indexers_test.go b/libbeat/processors/add_kubernetes_metadata/indexers_test.go index acbe1bf51eb..278275f10fd 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers_test.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers_test.go @@ -257,7 +257,7 @@ func TestFilteredGenMeta(t *testing.T) { assert.Equal(t, ok, true) assert.Equal(t, len(labelMap), 2) - rawAnnotations, _ := indexers[0].Data.GetValue("pod.annotations") + rawAnnotations, _ := indexers[0].Data.GetValue("annotations") assert.Nil(t, rawAnnotations) config, err := common.NewConfigFrom(map[string]interface{}{ @@ -284,7 +284,7 @@ func TestFilteredGenMeta(t *testing.T) { ok, _ = labelMap.HasKey("foo") assert.Equal(t, ok, true) - rawAnnotations, _ = indexers[0].Data.GetValue("pod.annotations") + rawAnnotations, _ = indexers[0].Data.GetValue("annotations") assert.NotNil(t, rawAnnotations) annotationsMap, ok := rawAnnotations.(common.MapStr) From 5f8b6606f4aa935c64b3552376b2dc17364a6252 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 25 Feb 2020 16:11:15 +0200 Subject: [PATCH 2/2] Add changelog entry Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ef7c8514f3e..05cc8f9e666 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717] - Fix loading processors from annotation hints. {pull}16348[16348] - Fix k8s pods labels broken schema. {pull}16480[16480] +- Fix k8s pods annotations broken schema. {pull}16554[16554] - Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450} *Auditbeat*