Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CONTP-277] add kubernetes_resources_annotations_as_tags and kubernetes_resources_labels_as_tags #27369

Merged
merged 5 commits into from
Aug 20, 2024
Merged

[CONTP-277] add kubernetes_resources_annotations_as_tags and kubernetes_resources_labels_as_tags #27369

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c732acd
add kubernetes_resources_annotations_as_tags and kubernetes_resources…
adel121 Aug 13, 2024
09faede
add release notes
adel121 Aug 13, 2024
52542e4
fix lint
adel121 Aug 13, 2024
dec764e
avoid computing metadata as tags each time in hostinfo tag extractor
adel121 Aug 16, 2024
9042bdd
add dummy kubenodeprovider in no_tags
adel121 Aug 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 75 additions & 22 deletions comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"

k8smetadata "github.com/DataDog/datadog-agent/comp/core/tagger/k8s_metadata"
Expand Down Expand Up @@ -114,8 +113,6 @@ var (
highCardOrchestratorLabels = map[string]string{
"io.rancher.container.name": tags.RancherContainer,
}

handledKubernetesMetadataResources = []string{"namespaces", "nodes"}
)

func (c *WorkloadMetaCollector) processEvents(evBundle workloadmeta.EventBundle) {
Expand Down Expand Up @@ -156,7 +153,7 @@ func (c *WorkloadMetaCollector) processEvents(evBundle workloadmeta.EventBundle)
case workloadmeta.KindProcess:
// tagInfos = append(tagInfos, c.handleProcess(ev)...) No tags for now
case workloadmeta.KindKubernetesDeployment:
// tagInfos = append(tagInfos, c.handleDeployment(ev)...) No tags for now
tagInfos = append(tagInfos, c.handleKubeDeployment(ev)...)
default:
log.Errorf("cannot handle event for entity %q with kind %q", entityID.ID, entityID.Kind)
}
Expand Down Expand Up @@ -346,16 +343,24 @@ func (c *WorkloadMetaCollector) handleKubePod(ev workloadmeta.Event) []*types.Ta

c.extractTagsFromPodLabels(pod, tagList)

// pod labels as tags
for name, value := range pod.Labels {
k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesLabelsAsTags["pods"], c.globK8sResourcesLabels["pods"], tagList)
}

// pod annotations as tags
for name, value := range pod.Annotations {
k8smetadata.AddMetadataAsTags(name, value, c.annotationsAsTags, c.globAnnotations, tagList)
k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesAnnotationsAsTags["pods"], c.globK8sResourcesAnnotations["pods"], tagList)
}

// namespace labels as tags
for name, value := range pod.NamespaceLabels {
k8smetadata.AddMetadataAsTags(name, value, c.nsLabelsAsTags, c.globNsLabels, tagList)
k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesLabelsAsTags["namespaces"], c.globK8sResourcesLabels["namespaces"], tagList)
}

// namespace annotations as tags
for name, value := range pod.NamespaceAnnotations {
k8smetadata.AddMetadataAsTags(name, value, c.nsAnnotationsAsTags, c.globNsAnnotations, tagList)
k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesAnnotationsAsTags["namespaces"], c.globK8sResourcesAnnotations["namespaces"], tagList)
}

kubeServiceDisabled := false
Expand Down Expand Up @@ -513,31 +518,79 @@ func (c *WorkloadMetaCollector) handleGardenContainer(container *workloadmeta.Co
}
}

func (c *WorkloadMetaCollector) handleKubeMetadata(ev workloadmeta.Event) []*types.TagInfo {
kubeMetadata := ev.Entity.(*workloadmeta.KubernetesMetadata)
func (c *WorkloadMetaCollector) handleKubeDeployment(ev workloadmeta.Event) []*types.TagInfo {
deployment := ev.Entity.(*workloadmeta.KubernetesDeployment)

resource := kubeMetadata.GVR.Resource
groupResource := "deployments.apps"

if !slices.Contains(handledKubernetesMetadataResources, resource) {
labelsAsTags := c.k8sResourcesLabelsAsTags[groupResource]
annotationsAsTags := c.k8sResourcesAnnotationsAsTags[groupResource]

if len(labelsAsTags)+len(annotationsAsTags) == 0 {
return nil
}

globLabels := c.globK8sResourcesLabels[groupResource]
globAnnotations := c.globK8sResourcesAnnotations[groupResource]

tagList := taglist.NewTagList()

switch resource {
case "nodes":
// No tags for nodes
case "namespaces":
for name, value := range kubeMetadata.Labels {
k8smetadata.AddMetadataAsTags(name, value, c.nsLabelsAsTags, c.globNsLabels, tagList)
}
for name, value := range deployment.Labels {
k8smetadata.AddMetadataAsTags(name, value, labelsAsTags, globLabels, tagList)
}

for name, value := range kubeMetadata.Annotations {
k8smetadata.AddMetadataAsTags(name, value, c.nsAnnotationsAsTags, c.globNsAnnotations, tagList)
}
for name, value := range deployment.Annotations {
k8smetadata.AddMetadataAsTags(name, value, annotationsAsTags, globAnnotations, tagList)
}

low, orch, high, standard := tagList.Compute()

if len(low)+len(orch)+len(high)+len(standard) == 0 {
return nil
}

tagInfos := []*types.TagInfo{
{
Source: deploymentSource,
Entity: buildTaggerEntityID(deployment.EntityID),
HighCardTags: high,
OrchestratorCardTags: orch,
LowCardTags: low,
StandardTags: standard,
},
}

return tagInfos
}

func (c *WorkloadMetaCollector) handleKubeMetadata(ev workloadmeta.Event) []*types.TagInfo {
kubeMetadata := ev.Entity.(*workloadmeta.KubernetesMetadata)

tagList := taglist.NewTagList()

// Generic resource annotations and labels as tags
groupResource := kubeMetadata.GVR.GroupResource().String()

labelsAsTags := c.k8sResourcesLabelsAsTags[groupResource]
annotationsAsTags := c.k8sResourcesAnnotationsAsTags[groupResource]

globLabels := c.globK8sResourcesLabels[groupResource]
globAnnotations := c.globK8sResourcesAnnotations[groupResource]

for name, value := range kubeMetadata.Labels {
k8smetadata.AddMetadataAsTags(name, value, labelsAsTags, globLabels, tagList)
}

for name, value := range kubeMetadata.Annotations {
k8smetadata.AddMetadataAsTags(name, value, annotationsAsTags, globAnnotations, tagList)
}

low, orch, high, standard := tagList.Compute()

if len(low)+len(orch)+len(high)+len(standard) == 0 {
return nil
}

tagInfos := []*types.TagInfo{
{
Source: kubeMetadataSource,
Expand Down Expand Up @@ -575,7 +628,7 @@ func (c *WorkloadMetaCollector) extractTagsFromPodLabels(pod *workloadmeta.Kuber
tagList.AddLow(tags.KubeAppManagedBy, value)
}

k8smetadata.AddMetadataAsTags(name, value, c.labelsAsTags, c.globLabels, tagList)
k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesLabelsAsTags["pods"], c.globK8sResourcesLabels["pods"], tagList)
}
}

Expand Down
46 changes: 25 additions & 21 deletions comp/core/tagger/taggerimpl/collectors/workloadmeta_main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/DataDog/datadog-agent/comp/core/tagger/types"
workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
"github.com/DataDog/datadog-agent/pkg/config"
configutils "github.com/DataDog/datadog-agent/pkg/config/utils"
"github.com/DataDog/datadog-agent/pkg/status/health"
"github.com/DataDog/datadog-agent/pkg/util"
"github.com/DataDog/datadog-agent/pkg/util/flavor"
Expand All @@ -34,6 +35,7 @@ const (
processSource = workloadmetaCollectorName + "-" + string(workloadmeta.KindProcess)
hostSource = workloadmetaCollectorName + "-" + string(workloadmeta.KindHost)
kubeMetadataSource = workloadmetaCollectorName + "-" + string(workloadmeta.KindKubernetesMetadata)
deploymentSource = workloadmetaCollectorName + "-" + string(workloadmeta.KindKubernetesDeployment)

clusterTagNamePrefix = "kube_cluster_name"
)
Expand All @@ -55,17 +57,13 @@ type WorkloadMetaCollector struct {
containerEnvAsTags map[string]string
containerLabelsAsTags map[string]string

staticTags map[string]string
labelsAsTags map[string]string
annotationsAsTags map[string]string
nsLabelsAsTags map[string]string
nsAnnotationsAsTags map[string]string
globLabels map[string]glob.Glob
globAnnotations map[string]glob.Glob
globNsLabels map[string]glob.Glob
globNsAnnotations map[string]glob.Glob
globContainerLabels map[string]glob.Glob
globContainerEnvLabels map[string]glob.Glob
staticTags map[string]string
k8sResourcesAnnotationsAsTags map[string]map[string]string
k8sResourcesLabelsAsTags map[string]map[string]string
globContainerLabels map[string]glob.Glob
globContainerEnvLabels map[string]glob.Glob
globK8sResourcesAnnotations map[string]map[string]glob.Glob
globK8sResourcesLabels map[string]map[string]glob.Glob

collectEC2ResourceTags bool
collectPersistentVolumeClaimsTags bool
Expand All @@ -76,11 +74,19 @@ func (c *WorkloadMetaCollector) initContainerMetaAsTags(labelsAsTags, envAsTags
c.containerEnvAsTags, c.globContainerEnvLabels = k8smetadata.InitMetadataAsTags(envAsTags)
}

func (c *WorkloadMetaCollector) initPodMetaAsTags(labelsAsTags, annotationsAsTags, nsLabelsAsTags, nsAnnotationsAsTags map[string]string) {
c.labelsAsTags, c.globLabels = k8smetadata.InitMetadataAsTags(labelsAsTags)
c.annotationsAsTags, c.globAnnotations = k8smetadata.InitMetadataAsTags(annotationsAsTags)
c.nsLabelsAsTags, c.globNsLabels = k8smetadata.InitMetadataAsTags(nsLabelsAsTags)
c.nsAnnotationsAsTags, c.globNsAnnotations = k8smetadata.InitMetadataAsTags(nsAnnotationsAsTags)
func (c *WorkloadMetaCollector) initK8sResourcesMetaAsTags(resourcesLabelsAsTags, resourcesAnnotationsAsTags map[string]map[string]string) {
c.k8sResourcesAnnotationsAsTags = map[string]map[string]string{}
c.k8sResourcesLabelsAsTags = map[string]map[string]string{}
c.globK8sResourcesAnnotations = map[string]map[string]glob.Glob{}
c.globK8sResourcesLabels = map[string]map[string]glob.Glob{}

for resource, labelsAsTags := range resourcesLabelsAsTags {
c.k8sResourcesLabelsAsTags[resource], c.globK8sResourcesLabels[resource] = k8smetadata.InitMetadataAsTags(labelsAsTags)
}

for resource, annotationsAsTags := range resourcesAnnotationsAsTags {
c.k8sResourcesAnnotationsAsTags[resource], c.globK8sResourcesAnnotations[resource] = k8smetadata.InitMetadataAsTags(annotationsAsTags)
}
}

// Run runs the continuous event watching loop and sends new tags to the
Expand Down Expand Up @@ -178,11 +184,9 @@ func NewWorkloadMetaCollector(_ context.Context, store workloadmeta.Component, p
)
c.initContainerMetaAsTags(containerLabelsAsTags, containerEnvAsTags)

labelsAsTags := config.Datadog().GetStringMapString("kubernetes_pod_labels_as_tags")
annotationsAsTags := config.Datadog().GetStringMapString("kubernetes_pod_annotations_as_tags")
nsLabelsAsTags := config.Datadog().GetStringMapString("kubernetes_namespace_labels_as_tags")
nsAnnotationsAsTags := config.Datadog().GetStringMapString("kubernetes_namespace_annotations_as_tags")
c.initPodMetaAsTags(labelsAsTags, annotationsAsTags, nsLabelsAsTags, nsAnnotationsAsTags)
// kubernetes resources metadata as tags
metadataAsTags := configutils.GetMetadataAsTags(config.Datadog())
c.initK8sResourcesMetaAsTags(metadataAsTags.GetResourcesLabelsAsTags(), metadataAsTags.GetResourcesAnnotationsAsTags())

return c
}
Expand Down
Loading
Loading