From 25c36935fb59092ce043199af426b9e9653da738 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Fri, 18 Nov 2022 14:58:19 +0100 Subject: [PATCH 1/3] add namespace MetaGen to the generic Resource Signed-off-by: Tetiana Kravchenko --- kubernetes/metadata/pod.go | 21 ++--- kubernetes/metadata/resource.go | 34 +++++-- kubernetes/metadata/resource_test.go | 132 +++++++++++++++++++++++++++ kubernetes/metadata/service.go | 32 +++---- 4 files changed, 178 insertions(+), 41 deletions(-) diff --git a/kubernetes/metadata/pod.go b/kubernetes/metadata/pod.go index af7b3f2b8a..02009b2943 100644 --- a/kubernetes/metadata/pod.go +++ b/kubernetes/metadata/pod.go @@ -33,7 +33,6 @@ type pod struct { store cache.Store client k8s.Interface node MetaGen - namespace MetaGen resource *Resource addResourceMetadata *AddResourceMetadataConfig } @@ -48,10 +47,9 @@ func NewPodMetadataGenerator( addResourceMetadata *AddResourceMetadataConfig) MetaGen { return &pod{ - resource: NewResourceMetadataGenerator(cfg, client), + resource: NewNamespaceAwareResourceMetadataGenerator(cfg, client, namespace), store: pods, node: node, - namespace: namespace, client: client, addResourceMetadata: addResourceMetadata, } @@ -59,10 +57,12 @@ func NewPodMetadataGenerator( // Generate generates pod metadata from a resource object // Metadata map is in the following form: -// { -// "kubernetes": {}, -// "some.ecs.field": "asdf" -// } +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// // All Kubernetes fields that need to be stored under kubernetes. prefix are populated by // GenerateK8s method while fields that are part of ECS are generated by GenerateECS method func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { @@ -122,13 +122,6 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr. _, _ = out.Put("node.name", po.Spec.NodeName) } - if p.namespace != nil { - meta := p.namespace.GenerateFromName(po.GetNamespace()) - if meta != nil { - out.DeepUpdate(meta) - } - } - if po.Status.PodIP != "" { _, _ = out.Put("pod.ip", po.Status.PodIP) } diff --git a/kubernetes/metadata/resource.go b/kubernetes/metadata/resource.go index 67f180c3c2..eb2ad0905e 100644 --- a/kubernetes/metadata/resource.go +++ b/kubernetes/metadata/resource.go @@ -36,6 +36,7 @@ const deploymentType = "Deployment" type Resource struct { config *Config clusterInfo ClusterInfo + namespace MetaGen } // NewResourceMetadataGenerator creates a metadata generator for a generic resource @@ -56,14 +57,23 @@ func NewResourceMetadataGenerator(cfg *config.C, client k8s.Interface) *Resource return r } +// NewNamespaceAwareResourceMetadataGenerator creates a metadata generator with informatuon about namespace +func NewNamespaceAwareResourceMetadataGenerator(cfg *config.C, client k8s.Interface, namespace MetaGen) *Resource { + r := NewResourceMetadataGenerator(cfg, client) + r.namespace = namespace + return r +} + // Generate generates metadata from a resource object // Generate method returns metadata in the following form: -// { -// "kubernetes": {}, -// "ecs.a.field": 42, -// } +// +// { +// "kubernetes": {}, +// "ecs.a.field": 42, +// } +// // This method should be called in top level and not as part of other metadata generators. -// For retrieving metadata without kubernetes. prefix one should call GenerateK8s instead. +// For retrieving metadata without 'kubernetes.' prefix one should call GenerateK8s instead. func (r *Resource) Generate(kind string, obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { ecsFields := r.GenerateECS(obj) meta := mapstr.M{ @@ -113,8 +123,18 @@ func (r *Resource) GenerateK8s(kind string, obj kubernetes.Resource, options ... }, } - if accessor.GetNamespace() != "" { - _ = safemapstr.Put(meta, "namespace", accessor.GetNamespace()) + namespaceName := accessor.GetNamespace() + if namespaceName != "" { + _ = safemapstr.Put(meta, "namespace", namespaceName) + + if r.namespace != nil { + nsMeta := r.namespace.GenerateFromName(namespaceName) + if nsMeta != nil { + // Use this in 8.0 + //out.Put("namespace", meta["namespace"]) + meta.DeepUpdate(nsMeta) + } + } } // Add controller metadata if present diff --git a/kubernetes/metadata/resource_test.go b/kubernetes/metadata/resource_test.go index 7ad7affcb6..434b6b33c9 100644 --- a/kubernetes/metadata/resource_test.go +++ b/kubernetes/metadata/resource_test.go @@ -22,11 +22,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/go-ucfg" ) @@ -130,3 +134,131 @@ func TestResource_Generate(t *testing.T) { }) } } + +func TestNamespaceAwareResource_GenerateWithNamespace(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + resourceName string + input kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test not namespaced kubernetes resource - PersistentVolume", + resourceName: "persistentvolume", + input: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-18705cfb-9fb8-441f-9b32-0d67a21af839", + UID: "020fd954-3674-496a-9e77-c25f0f2257ea", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolume", + APIVersion: "v1", + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{ + "ns.annotation": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "persistentvolume": mapstr.M{ + "name": "pvc-18705cfb-9fb8-441f-9b32-0d67a21af839", + "uid": "020fd954-3674-496a-9e77-c25f0f2257ea", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + }, + }, + }, + { + name: "test namespaced kubernetes resource", + resourceName: "deployment", + input: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + UID: "f33ca314-8cc5-48ea-90b7-3102c7430f75", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{ + "ns.annotation": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "deployment": mapstr.M{ + "name": name, + "uid": "f33ca314-8cc5-48ea-90b7-3102c7430f75", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": "default", + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "nskey": "nsvalue", + }, + "namespace_annotations": mapstr.M{ + "ns_annotation": "value", + }, + }, + }, + }, + } + + for _, test := range tests { + nsConfig, err := config.NewConfigFrom(map[string]interface{}{ + "include_annotations": []string{"ns.annotation"}, + }) + require.NoError(t, err) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(nsConfig, namespaces, client) + + metagen := NewNamespaceAwareResourceMetadataGenerator(nsConfig, client, nsMeta) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.resourceName, test.input)) + }) + } +} diff --git a/kubernetes/metadata/service.go b/kubernetes/metadata/service.go index 11deb823fc..ed64ca2257 100644 --- a/kubernetes/metadata/service.go +++ b/kubernetes/metadata/service.go @@ -28,26 +28,26 @@ import ( ) type service struct { - store cache.Store - namespace MetaGen - resource *Resource + store cache.Store + resource *Resource } -// NewServiceMetadataGenerator creates a metagen for service resources +// NewServiceMetadataGenerator creates a metagen for namespaced Kubernetes resources, like: Service, Deployment, Cronjob, etc. func NewServiceMetadataGenerator(cfg *config.C, services cache.Store, namespace MetaGen, client k8s.Interface) MetaGen { return &service{ - resource: NewResourceMetadataGenerator(cfg, client), - store: services, - namespace: namespace, + resource: NewNamespaceAwareResourceMetadataGenerator(cfg, client, namespace), + store: services, } } // Generate generates service metadata from a resource object // Metadata map is in the following form: -// { -// "kubernetes": {}, -// "some.ecs.field": "asdf" -// } +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// // All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by // GenerateK8s method while fields that are part of ECS are generated by GenerateECS method func (s *service) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { @@ -71,17 +71,9 @@ func (s *service) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) map return nil } + // resourceName := obj.GetObjectKind().GroupVersionKind().Kind out := s.resource.GenerateK8s("service", obj, opts...) - if s.namespace != nil { - meta := s.namespace.GenerateFromName(svc.GetNamespace()) - if meta != nil { - // Use this in 8.0 - //out.Put("namespace", meta["namespace"]) - out.DeepUpdate(meta) - } - } - selectors := svc.Spec.Selector if len(selectors) == 0 { return out From f5aa293fac068a6d709ee946d5108613f01bd0ef Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Fri, 18 Nov 2022 15:02:19 +0100 Subject: [PATCH 2/3] clean up some local tests adjustments Signed-off-by: Tetiana Kravchenko --- kubernetes/metadata/service.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kubernetes/metadata/service.go b/kubernetes/metadata/service.go index ed64ca2257..812aacede8 100644 --- a/kubernetes/metadata/service.go +++ b/kubernetes/metadata/service.go @@ -32,7 +32,7 @@ type service struct { resource *Resource } -// NewServiceMetadataGenerator creates a metagen for namespaced Kubernetes resources, like: Service, Deployment, Cronjob, etc. +// NewServiceMetadataGenerator creates a metagen for service resources func NewServiceMetadataGenerator(cfg *config.C, services cache.Store, namespace MetaGen, client k8s.Interface) MetaGen { return &service{ resource: NewNamespaceAwareResourceMetadataGenerator(cfg, client, namespace), @@ -71,7 +71,6 @@ func (s *service) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) map return nil } - // resourceName := obj.GetObjectKind().GroupVersionKind().Kind out := s.resource.GenerateK8s("service", obj, opts...) selectors := svc.Spec.Selector From bec6581d6e9e1b64cc8b97c57681b63c0390b4c3 Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Mon, 21 Nov 2022 15:14:53 +0100 Subject: [PATCH 3/3] remove comment Signed-off-by: Tetiana Kravchenko --- kubernetes/metadata/resource.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/kubernetes/metadata/resource.go b/kubernetes/metadata/resource.go index eb2ad0905e..44ebdaa718 100644 --- a/kubernetes/metadata/resource.go +++ b/kubernetes/metadata/resource.go @@ -130,8 +130,6 @@ func (r *Resource) GenerateK8s(kind string, obj kubernetes.Resource, options ... if r.namespace != nil { nsMeta := r.namespace.GenerateFromName(namespaceName) if nsMeta != nil { - // Use this in 8.0 - //out.Put("namespace", meta["namespace"]) meta.DeepUpdate(nsMeta) } }