OTEL-738: Add Custom Container Tag Support (datadog.container.tag pre…

…fix) (#193)

* OTEL-738: Add Custom Container Tag Support (datadog.container.tag prefix)

* add changelog

* lint

* Change customContainerTagPrefix to const

* pluralize func

* address feedback

* refactor

* run make fmt

* use .Str()

.chloggen/mackjmr_add-custom-container-tag-support-deprecate.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: deprecation
+
+# The name of the component (e.g. pkg/quantile)
+component: pkg/otlp/attributes
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: ContainerTagFromAttributes is deprecated in favor of ContainerTagFromResourceAttributes.
+
+# The PR related to this change
+issues: [193]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

.chloggen/mackjmr_add-custom-container-tag-support.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component (e.g. pkg/quantile)
+component: pkg/otlp/attributes
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for custom container tags via resource attribute prefix `datadog.container.tag`.
+
+# The PR related to this change
+issues: [193]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

pkg/otlp/attributes/attributes.go

@@ -16,21 +16,29 @@ package attributes
 
 import (
 	"fmt"
+	"strings"
 
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	conventions "go.opentelemetry.io/collector/semconv/v1.6.1"
 )
 
+// customContainerTagPrefix defines the prefix for custom container tags.
+const customContainerTagPrefix = "datadog.container.tag."
+
 var (
-	// conventionsMappings defines the mapping between OpenTelemetry semantic conventions
-	// and Datadog Agent conventions
-	conventionsMapping = map[string]string{
+	// coreMapping defines the mapping between OpenTelemetry semantic conventions
+	// and Datadog Agent conventions for env, service and version.
+	coreMapping = map[string]string{
 		// Datadog conventions
 		// https://docs.datadoghq.com/getting_started/tagging/unified_service_tagging/
 		conventions.AttributeDeploymentEnvironment: "env",
 		conventions.AttributeServiceName:           "service",
 		conventions.AttributeServiceVersion:        "version",
+	}
 
+	// containerMappings defines the mapping between OpenTelemetry semantic conventions
+	// and Datadog Agent conventions for containers.
+	containerMappings = map[string]string{
 		// Containers
 		conventions.AttributeContainerID:        "container_id",
 		conventions.AttributeContainerName:      "container_name",
@@ -66,33 +74,6 @@ var (
 		conventions.AttributeK8SPodName:         "pod_name",
 	}
 
-	// containerTagsAttributes contains a set of attributes that will be extracted as Datadog container tags.
-	containerTagsAttributes = []string{
-		conventions.AttributeContainerID,
-		conventions.AttributeContainerName,
-		conventions.AttributeContainerImageName,
-		conventions.AttributeContainerImageTag,
-		conventions.AttributeContainerRuntime,
-		conventions.AttributeK8SContainerName,
-		conventions.AttributeK8SClusterName,
-		conventions.AttributeK8SDeploymentName,
-		conventions.AttributeK8SReplicaSetName,
-		conventions.AttributeK8SStatefulSetName,
-		conventions.AttributeK8SDaemonSetName,
-		conventions.AttributeK8SJobName,
-		conventions.AttributeK8SCronJobName,
-		conventions.AttributeK8SNamespaceName,
-		conventions.AttributeK8SPodName,
-		conventions.AttributeCloudProvider,
-		conventions.AttributeCloudRegion,
-		conventions.AttributeCloudAvailabilityZone,
-		conventions.AttributeAWSECSTaskFamily,
-		conventions.AttributeAWSECSTaskARN,
-		conventions.AttributeAWSECSClusterARN,
-		conventions.AttributeAWSECSTaskRevision,
-		conventions.AttributeAWSECSContainerARN,
-	}
-
 	// Kubernetes mappings defines the mapping between Kubernetes conventions (both general and Datadog specific)
 	// and Datadog Agent conventions. The Datadog Agent conventions can be found at
 	// https://github.com/DataDog/datadog-agent/blob/e081bed/pkg/tagger/collectors/const.go and
@@ -142,8 +123,8 @@ func TagsFromAttributes(attrs pcommon.Map) []string {
 			systemAttributes.OSType = value.Str()
 		}
 
-		// conventions mapping
-		if datadogKey, found := conventionsMapping[key]; found && value.Str() != "" {
+		// core attributes mapping
+		if datadogKey, found := coreMapping[key]; found && value.Str() != "" {
 			tags = append(tags, fmt.Sprintf("%s:%s", datadogKey, value.Str()))
 		}
 
@@ -154,6 +135,12 @@ func TagsFromAttributes(attrs pcommon.Map) []string {
 		return true
 	})
 
+	// Container Tag mappings
+	ctags := ContainerTagsFromResourceAttributes(attrs)
+	for key, val := range ctags {
+		tags = append(tags, fmt.Sprintf("%s:%s", key, val))
+	}
+
 	tags = append(tags, processAttributes.extractTags()...)
 	tags = append(tags, systemAttributes.extractTags()...)
 
@@ -173,16 +160,47 @@ func OriginIDFromAttributes(attrs pcommon.Map) (originID string) {
 	return
 }
 
+// ContainerTagFromResourceAttributes extracts container tags from the given
+// set of resource attributes. Container tags are extracted via semantic
+// conventions. Customer container tags are extracted via resource attributes
+// prefixed by datadog.container.tag. Custom container tag values of a different type
+// than ValueTypeStr will be ignored.
+// In the case of duplicates between semantic conventions and custom resource attributes
+// (e.g. container.id, datadog.container.tag.container_id) the semantic convention takes
+// precedence.
+func ContainerTagsFromResourceAttributes(attrs pcommon.Map) map[string]string {
+	ddtags := make(map[string]string)
+	attrs.Range(func(key string, value pcommon.Value) bool {
+		// Semantic Conventions
+		if datadogKey, found := containerMappings[key]; found && value.Str() != "" {
+			ddtags[datadogKey] = value.Str()
+		}
+		// Custom (datadog.container.tag namespace)
+		if strings.HasPrefix(key, customContainerTagPrefix) {
+			customKey := strings.TrimPrefix(key, customContainerTagPrefix)
+			if customKey != "" && value.Str() != "" {
+				// Do not replace if set via semantic conventions mappings.
+				if _, found := ddtags[customKey]; !found {
+					ddtags[customKey] = value.Str()
+				}
+			}
+		}
+		return true
+	})
+	return ddtags
+}
+
 // ContainerTagFromAttributes extracts the value of _dd.tags.container from the given
 // set of attributes.
+// Deprecated: Deprecated in favor of ContainerTagFromResourceAttributes.
 func ContainerTagFromAttributes(attr map[string]string) map[string]string {
 	ddtags := make(map[string]string)
-	for _, key := range containerTagsAttributes {
-		val, ok := attr[key]
-		if !ok {
+	for key, val := range attr {
+		datadogKey, found := containerMappings[key]
+		if !found {
 			continue
 		}
-		ddtags[conventionsMapping[key]] = val
+		ddtags[datadogKey] = val
 	}
 	return ddtags
 }

pkg/otlp/attributes/attributes_test.go

@@ -36,6 +36,9 @@ func TestTagsFromAttributes(t *testing.T) {
 		conventions.AttributeAWSECSClusterARN:      "cluster_arn",
 		conventions.AttributeContainerRuntime:      "cro",
 		"tags.datadoghq.com/service":               "service_name",
+		conventions.AttributeDeploymentEnvironment: "prod",
+		conventions.AttributeContainerName:         "custom",
+		"datadog.container.tag.custom.team":        "otel",
 	}
 	attrs := pcommon.NewMap()
 	attrs.FromRaw(attributeMap)
@@ -47,6 +50,9 @@ func TestTagsFromAttributes(t *testing.T) {
 		fmt.Sprintf("%s:%s", "ecs_cluster_name", "cluster_arn"),
 		fmt.Sprintf("%s:%s", "service", "service_name"),
 		fmt.Sprintf("%s:%s", "runtime", "cro"),
+		fmt.Sprintf("%s:%s", "env", "prod"),
+		fmt.Sprintf("%s:%s", "container_name", "custom"),
+		fmt.Sprintf("%s:%s", "custom.team", "otel"),
 	}, TagsFromAttributes(attrs))
 }
 
@@ -56,6 +62,76 @@ func TestTagsFromAttributesEmpty(t *testing.T) {
 	assert.Equal(t, []string{}, TagsFromAttributes(attrs))
 }
 
+func TestContainerTagFromResourceAttributes(t *testing.T) {
+	t.Run("valid", func(t *testing.T) {
+		attributes := pcommon.NewMap()
+		err := attributes.FromRaw(map[string]interface{}{
+			conventions.AttributeContainerName:         "sample_app",
+			conventions.AttributeContainerImageTag:     "sample_app_image_tag",
+			conventions.AttributeContainerRuntime:      "cro",
+			conventions.AttributeK8SContainerName:      "kube_sample_app",
+			conventions.AttributeK8SReplicaSetName:     "sample_replica_set",
+			conventions.AttributeK8SDaemonSetName:      "sample_daemonset_name",
+			conventions.AttributeK8SPodName:            "sample_pod_name",
+			conventions.AttributeCloudProvider:         "sample_cloud_provider",
+			conventions.AttributeCloudRegion:           "sample_region",
+			conventions.AttributeCloudAvailabilityZone: "sample_zone",
+			conventions.AttributeAWSECSTaskFamily:      "sample_task_family",
+			conventions.AttributeAWSECSClusterARN:      "sample_ecs_cluster_name",
+			conventions.AttributeAWSECSContainerARN:    "sample_ecs_container_name",
+			"datadog.container.tag.custom.team":        "otel",
+		})
+		assert.NoError(t, err)
+		assert.Equal(t, map[string]string{
+			"container_name":      "sample_app",
+			"image_tag":           "sample_app_image_tag",
+			"runtime":             "cro",
+			"kube_container_name": "kube_sample_app",
+			"kube_replica_set":    "sample_replica_set",
+			"kube_daemon_set":     "sample_daemonset_name",
+			"pod_name":            "sample_pod_name",
+			"cloud_provider":      "sample_cloud_provider",
+			"region":              "sample_region",
+			"zone":                "sample_zone",
+			"task_family":         "sample_task_family",
+			"ecs_cluster_name":    "sample_ecs_cluster_name",
+			"ecs_container_name":  "sample_ecs_container_name",
+			"custom.team":         "otel",
+		}, ContainerTagsFromResourceAttributes(attributes))
+		fmt.Println(ContainerTagsFromResourceAttributes(attributes))
+	})
+
+	t.Run("conventions vs custom", func(t *testing.T) {
+		attributes := pcommon.NewMap()
+		err := attributes.FromRaw(map[string]interface{}{
+			conventions.AttributeContainerName:     "ok",
+			"datadog.container.tag.container_name": "nok",
+		})
+		assert.NoError(t, err)
+		assert.Equal(t, map[string]string{
+			"container_name": "ok",
+		}, ContainerTagsFromResourceAttributes(attributes))
+	})
+
+	t.Run("invalid", func(t *testing.T) {
+		attributes := pcommon.NewMap()
+		err := attributes.FromRaw(map[string]interface{}{
+			"empty_string_val": "",
+			"":                 "empty_string_key",
+			"custom_tag":       "example_custom_tag",
+		})
+		assert.NoError(t, err)
+		slice := attributes.PutEmptySlice("datadog.container.tag.slice")
+		slice.AppendEmpty().SetStr("value1")
+		slice.AppendEmpty().SetStr("value2")
+		assert.Equal(t, map[string]string{}, ContainerTagsFromResourceAttributes(attributes))
+	})
+
+	t.Run("empty", func(t *testing.T) {
+		assert.Empty(t, ContainerTagsFromResourceAttributes(pcommon.NewMap()))
+	})
+}
+
 func TestContainerTagFromAttributes(t *testing.T) {
 	attributeMap := map[string]string{
 		conventions.AttributeContainerName:         "sample_app",