Enable sending host tags with metrics for a configurable duration

comp/core/tagger/collectors/workloadmeta_extract.go

@@ -24,6 +24,9 @@ const (
 	// GlobalEntityID defines the entity ID that holds global tags
 	GlobalEntityID = "internal://global-entity-id"
 
+	// HostEntityID defines the entity ID that holds host tags
+	HostEntityID = "internal://host-entity-id"
+
 	podAnnotationPrefix              = "ad.datadoghq.com/"
 	podContainerTagsAnnotationFormat = podAnnotationPrefix + "%s.tags"
 	podTagsAnnotation                = podAnnotationPrefix + "tags"

comp/core/tagger/collectors/workloadmeta_main.go

@@ -8,11 +8,13 @@ package collectors
 import (
 	"context"
 	"strings"
+	"time"
 
 	"github.com/gobwas/glob"
 
 	"github.com/DataDog/datadog-agent/comp/core/tagger/utils"
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta"
+	"github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags"
 	"github.com/DataDog/datadog-agent/pkg/config"
 	"github.com/DataDog/datadog-agent/pkg/status/health"
 	"github.com/DataDog/datadog-agent/pkg/util"
@@ -25,6 +27,7 @@ const (
 	workloadmetaCollectorName = "workloadmeta"
 
 	staticSource         = workloadmetaCollectorName + "-static"
+	hostSource           = workloadmetaCollectorName + "-host"
 	podSource            = workloadmetaCollectorName + "-" + string(workloadmeta.KindKubernetesPod)
 	nodeSource           = workloadmetaCollectorName + "-" + string(workloadmeta.KindKubernetesNode)
 	taskSource           = workloadmetaCollectorName + "-" + string(workloadmeta.KindECSTask)
@@ -76,6 +79,24 @@ func (c *WorkloadMetaCollector) initPodMetaAsTags(labelsAsTags, annotationsAsTag
 	c.nsLabelsAsTags, c.globNsLabels = utils.InitMetadataAsTags(nsLabelsAsTags)
 }
 
+func (c *WorkloadMetaCollector) injectHostTags() {
+	duration := config.Datadog.GetDuration("expected_tags_duration")
+	if duration <= 0 {
+		return
+	}
+	tags := hosttags.GetHostTags(context.TODO(), false, config.Datadog).System
+	log.Debugf("Adding host tags to metrics for %v : %v", duration, tags)
+
+	c.tagProcessor.ProcessTagInfo([]*TagInfo{
+		{
+			Source:      hostSource,
+			Entity:      HostEntityID,
+			LowCardTags: tags,
+			ExpiryDate:  time.Now().Add(duration), // Ensure host tags are expired after the configured interval
+		},
+	})
+}
+
 // Run runs the continuous event watching loop and sends new tags to the
 // tagger based on the events sent by the workloadmeta.
 func (c *WorkloadMetaCollector) Run(ctx context.Context) {
@@ -174,6 +195,7 @@ func NewWorkloadMetaCollector(_ context.Context, store workloadmeta.Component, p
 	annotationsAsTags := config.Datadog.GetStringMapString("kubernetes_pod_annotations_as_tags")
 	nsLabelsAsTags := config.Datadog.GetStringMapString("kubernetes_namespace_labels_as_tags")
 	c.initPodMetaAsTags(labelsAsTags, annotationsAsTags, nsLabelsAsTags)
+	c.injectHostTags()
 
 	return c
 }

comp/core/tagger/local/tagger.go

@@ -37,9 +37,9 @@ type Tagger struct {
 
 // NewTagger returns an allocated tagger. You are probably looking for
 // tagger.Tag() using the global instance instead of creating your own.
-func NewTagger(workloadStore workloadmeta.Component) *Tagger {
+func NewTagger(workloadStore workloadmeta.Component, tagStore *tagstore.TagStore) *Tagger {
 	return &Tagger{
-		tagStore:      tagstore.NewTagStore(),
+		tagStore:      tagStore,
 		workloadStore: workloadStore,
 	}
 }

comp/core/tagger/local/tagger_test.go

@@ -8,12 +8,16 @@ package local
 import (
 	"context"
 	"testing"
+	"time"
 
+	"github.com/benbjohnson/clock"
 	"github.com/stretchr/testify/assert"
 	"go.uber.org/fx"
 
 	"github.com/DataDog/datadog-agent/comp/core"
+	"github.com/DataDog/datadog-agent/comp/core/config"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/collectors"
+	"github.com/DataDog/datadog-agent/comp/core/tagger/tagstore"
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta"
 	"github.com/DataDog/datadog-agent/pkg/tagset"
 	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
@@ -26,7 +30,7 @@ func TestTagBuilder(t *testing.T) {
 		fx.Supply(workloadmeta.NewParams()),
 		workloadmeta.MockModuleV2(),
 	))
-	tagger := NewTagger(store)
+	tagger := NewTagger(store, tagstore.NewTagStore())
 	tagger.Start(context.Background())
 	defer tagger.Stop()
 
@@ -49,3 +53,40 @@ func TestTagBuilder(t *testing.T) {
 	assert.NoError(t, err)
 	assert.ElementsMatch(t, []string{"high", "low1", "low2"}, tb.Get())
 }
+
+func TestInjectHostTags(t *testing.T) {
+
+	overrides := map[string]interface{}{
+		"tags":                   []string{"tag1:value1", "tag2", "tag3"},
+		"expected_tags_duration": "10m",
+	}
+
+	store := fxutil.Test[workloadmeta.Mock](t, fx.Options(
+		fx.Replace(config.MockParams{Overrides: overrides}),
+		core.MockBundle(),
+		fx.Supply(workloadmeta.NewParams()),
+		workloadmeta.MockModuleV2(),
+	))
+
+	testClock := clock.NewMock()
+	testClock.Set(time.Now()) // Set the mock to now since the tag expire date is based off of the real clock time.
+
+	tagStore := tagstore.NewTagStoreWithClock(testClock)
+	tagger := NewTagger(store, tagStore)
+	tagger.Start(context.Background())
+	defer tagger.Stop()
+
+	tb := tagset.NewHashlessTagsAccumulator()
+
+	tagger.AccumulateTagsFor(collectors.HostEntityID, collectors.LowCardinality, tb)
+	assert.ElementsMatch(t, []string{"tag1:value1", "tag2", "tag3"}, tb.Get())
+
+	tb.Reset()
+	// Advance the clock by 11 minutes so prune will expire the tags.
+	testClock.Add(11 * time.Minute)
+	// Force a prune to remove the expired tags (this usually happens on a long timer).
+	tagStore.Prune()
+
+	tagger.AccumulateTagsFor(collectors.HostEntityID, collectors.LowCardinality, tb)
+	assert.Empty(t, tb.Get())
+}

comp/core/tagger/tagger.go

@@ -10,13 +10,16 @@ import (
 	"reflect"
 	"sync"
 
+	"go.uber.org/fx"
+
 	configComponent "github.com/DataDog/datadog-agent/comp/core/config"
 	logComp "github.com/DataDog/datadog-agent/comp/core/log"
 	tagger_api "github.com/DataDog/datadog-agent/comp/core/tagger/api"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/collectors"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/local"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/remote"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/replay"
+	"github.com/DataDog/datadog-agent/comp/core/tagger/tagstore"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/types"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/utils"
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta"
@@ -29,7 +32,6 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/util/fxutil"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 	"github.com/DataDog/datadog-agent/pkg/util/optional"
-	"go.uber.org/fx"
 )
 
 type dependencies struct {
@@ -117,7 +119,7 @@ func newTaggerClient(deps dependencies) Component {
 		}
 	case LocalTaggerAgent:
 		taggerClient = &TaggerClient{
-			defaultTagger: local.NewTagger(deps.Wmeta),
+			defaultTagger: local.NewTagger(deps.Wmeta, tagstore.NewTagStore()),
 			captureTagger: nil,
 		}
 	case FakeTagger:
@@ -291,6 +293,9 @@ func (t *TaggerClient) globalTagBuilder(cardinality collectors.TagCardinality, t
 		}
 	}
 	t.mux.RUnlock()
+
+	// Accumulate host tags - it's ok to ignore the error here since collecting host tags is an optional setting configured by the user.
+	_ = t.defaultTagger.AccumulateTagsFor(collectors.HostEntityID, cardinality, tb)
 	return t.defaultTagger.AccumulateTagsFor(collectors.GlobalEntityID, cardinality, tb)
 }
 

comp/core/tagger/tagstore/tagstore.go

@@ -49,10 +49,11 @@ type TagStore struct {
 
 // NewTagStore creates new TagStore.
 func NewTagStore() *TagStore {
-	return newTagStoreWithClock(clock.New())
+	return NewTagStoreWithClock(clock.New())
 }
 
-func newTagStoreWithClock(clock clock.Clock) *TagStore {
+// NewTagStoreWithClock is exported for testing
+func NewTagStoreWithClock(clock clock.Clock) *TagStore {
 	return &TagStore{
 		telemetry:  make(map[string]map[string]float64),
 		store:      make(map[string]*EntityTags),

comp/core/tagger/tagstore/tagstore_test.go

@@ -28,7 +28,7 @@ func (s *StoreTestSuite) SetupTest() {
 	s.clock = clock.NewMock()
 	// set the mock clock to the current time
 	s.clock.Add(time.Since(time.Unix(0, 0)))
-	s.store = newTagStoreWithClock(s.clock)
+	s.store = NewTagStoreWithClock(s.clock)
 }
 
 func (s *StoreTestSuite) TestIngest() {
@@ -356,7 +356,7 @@ type entityEventExpectation struct {
 
 func TestSubscribe(t *testing.T) {
 	clock := clock.NewMock()
-	store := newTagStoreWithClock(clock)
+	store := NewTagStoreWithClock(clock)
 
 	collectors.CollectorPriorities["source2"] = collectors.ClusterOrchestrator
 	collectors.CollectorPriorities["source"] = collectors.NodeRuntime

comp/metadata/host/hostimpl/utils/tags.go → comp/metadata/host/hostimpl/hosttags/tags.go

@@ -3,7 +3,8 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-package utils
+// Package hosttags provides access to host tags
+package hosttags
 
 import (
 	"context"

comp/metadata/host/hostimpl/utils/tags_test.go → comp/metadata/host/hostimpl/hosttags/tags_test.go

@@ -3,7 +3,8 @@
 // This product includes software developed at Datadog (https://www.datadoghq.com/).
 // Copyright 2016-present Datadog, Inc.
 
-package utils
+// Package hosttags provides access to host tags
+package hosttags
 
 import (
 	"context"

comp/metadata/host/hostimpl/utils/host.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags"
 	"github.com/DataDog/datadog-agent/comp/otelcol/otlp"
 	"github.com/DataDog/datadog-agent/pkg/collector/python"
 	"github.com/DataDog/datadog-agent/pkg/config"
@@ -90,7 +91,7 @@ type Payload struct {
 	PythonVersion string            `json:"python"`
 	SystemStats   *systemStats      `json:"systemStats"`
 	Meta          *Meta             `json:"meta"`
-	HostTags      *Tags             `json:"host-tags"`
+	HostTags      *hosttags.Tags    `json:"host-tags"`
 	ContainerMeta map[string]string `json:"container-meta,omitempty"`
 	NetworkMeta   *NetworkMeta      `json:"network"`
 	LogsMeta      *LogsMeta         `json:"logs"`
@@ -184,7 +185,7 @@ func GetPayload(ctx context.Context, conf config.Reader) *Payload {
 		PythonVersion: python.GetPythonInfo(),
 		SystemStats:   getSystemStats(),
 		Meta:          meta,
-		HostTags:      GetHostTags(ctx, false, conf),
+		HostTags:      hosttags.GetHostTags(ctx, false, conf),
 		ContainerMeta: containerMetadata.Get(1 * time.Second),
 		NetworkMeta:   getNetworkMeta(ctx),
 		LogsMeta:      getLogsMeta(conf),

pkg/cloudfoundry/containertagger/container_tagger.go

@@ -16,7 +16,7 @@ import (
 
 	"github.com/DataDog/datadog-agent/comp/core/tagger/utils"
 	"github.com/DataDog/datadog-agent/comp/core/workloadmeta"
-	hostMetadataUtils "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/utils"
+	hostMetadataUtils "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags"
 	"github.com/DataDog/datadog-agent/pkg/config"
 	"github.com/DataDog/datadog-agent/pkg/util/cloudproviders/cloudfoundry"
 	"github.com/DataDog/datadog-agent/pkg/util/log"

pkg/config/setup/config.go

@@ -20,6 +20,9 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v2"
+
 	"github.com/DataDog/datadog-agent/comp/core/secrets"
 	"github.com/DataDog/datadog-agent/pkg/collector/check/defaults"
 	pkgconfigenv "github.com/DataDog/datadog-agent/pkg/config/env"
@@ -28,8 +31,6 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 	"github.com/DataDog/datadog-agent/pkg/util/optional"
 	"github.com/DataDog/datadog-agent/pkg/util/system"
-	"golang.org/x/exp/slices"
-	"gopkg.in/yaml.v2"
 )
 
 const (
@@ -740,6 +741,10 @@ func InitConfig(config pkgconfigmodel.Config) {
 	// Used internally to protect against configurations where metadata endpoints return incorrect values with 200 status codes.
 	config.BindEnvAndSetDefault("metadata_endpoints_max_hostname_size", 255)
 
+	// Duration during which the host tags will be submitted with metrics.
+	// Note: this setting must be no less than minute increments.
+	config.BindEnvAndSetDefault("expected_tags_duration", time.Duration(0))
+
 	// EC2
 	config.BindEnvAndSetDefault("ec2_use_windows_prefix_detection", false)
 	config.BindEnvAndSetDefault("ec2_metadata_timeout", 300)          // value in milliseconds

pkg/logs/internal/tag/local_provider.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 
 	"github.com/DataDog/datadog-agent/comp/logs/agent/config"
-	hostMetadataUtils "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/utils"
+	hostMetadataUtils "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags"
 	coreConfig "github.com/DataDog/datadog-agent/pkg/config"
 
 	"github.com/benbjohnson/clock"

releasenotes/notes/add-host-tags-to-metrics-8705f7b52d28d5f7.yaml

@@ -0,0 +1,12 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+features:
+  - |
+    Attach host tags to metrics when ``expected_tags_duration`` is configured to 
+    a time interval greater than 0.