From 343d6aa463bae30e6e370058257ec3985926fc5f Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Tue, 14 May 2024 13:43:24 -0400 Subject: [PATCH 01/15] Changes defaultPeerTagList to be read from a file Part of larger changes to come where common tag names are sourced from a single source of truth across DD repos The planned source of truth will generate static files which can be read by the agent code and synchronized across all DD repos involving peer tags --- pkg/trace/stats/concentrator.go | 43 +++++++++------------------------ pkg/trace/stats/peer_tags.csv | 1 + 2 files changed, 13 insertions(+), 31 deletions(-) create mode 100644 pkg/trace/stats/peer_tags.csv diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index 06ff985c2778e..8038994507382 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -6,6 +6,7 @@ package stats import ( + _ "embed" "sort" "strings" "sync" @@ -55,37 +56,17 @@ type Concentrator struct { statsd statsd.ClientInterface } -var defaultPeerTags = []string{ - "_dd.base_service", - "amqp.destination", - "amqp.exchange", - "amqp.queue", - "aws.queue.name", - "bucketname", - "cassandra.cluster", - "db.cassandra.contact.points", - "db.couchbase.seed.nodes", - "db.hostname", - "db.instance", - "db.name", - "db.system", - "hazelcast.instance", - "messaging.kafka.bootstrap.servers", - "mongodb.db", - "msmq.queue.path", - "net.peer.name", - "network.destination.name", - "peer.hostname", - "peer.service", - "queuename", - "rpc.service", - "rulename", - "server.address", - "statemachinename", - "streamname", - "tablename", - "topicname", -} +//go:embed peer_tags.csv +var peer_tag_file string + +var defaultPeerTags = func() []string { + lines := strings.Split(strings.TrimSpace(peer_tag_file), "\n") + tags := make([]string, len(lines)) + for i, line := range lines { + tags[i] = strings.TrimSpace(line) + } + return tags +}() func preparePeerTags(tags ...string) []string { if len(tags) == 0 { diff --git a/pkg/trace/stats/peer_tags.csv b/pkg/trace/stats/peer_tags.csv new file mode 100644 index 0000000000000..28782949e6519 --- /dev/null +++ b/pkg/trace/stats/peer_tags.csv @@ -0,0 +1 @@ +"_dd.base_service","amqp.destination","amqp.exchange","amqp.queue","aws.queue.name","bucketname","cassandra.cluster","db.cassandra.contact.points","db.couchbase.seed.nodes","db.hostname","db.instance","db.name","db.system","hazelcast.instance","messaging.kafka.bootstrap.servers","mongodb.db","msmq.queue.path","net.peer.name","network.destination.name","peer.hostname","peer.service","queuename","rpc.service","rulename","server.address","statemachinename","streamname","tablename","topicname" From 5f7776284b11e23978999d5b432573b61d5695bf Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Tue, 14 May 2024 14:40:56 -0400 Subject: [PATCH 02/15] linter fixes --- pkg/trace/stats/concentrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index 8038994507382..d1810feeed911 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -57,7 +57,7 @@ type Concentrator struct { } //go:embed peer_tags.csv -var peer_tag_file string +var peerTagFile string var defaultPeerTags = func() []string { lines := strings.Split(strings.TrimSpace(peer_tag_file), "\n") From 5e2ad060a23ee8ae42f68ec516f7109563f6063e Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Tue, 14 May 2024 15:26:17 -0400 Subject: [PATCH 03/15] linter build fixes --- pkg/trace/stats/concentrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index d1810feeed911..0c6c10a5609a8 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -60,7 +60,7 @@ type Concentrator struct { var peerTagFile string var defaultPeerTags = func() []string { - lines := strings.Split(strings.TrimSpace(peer_tag_file), "\n") + lines := strings.Split(strings.TrimSpace(peerTagFile), "\n") tags := make([]string, len(lines)) for i, line := range lines { tags[i] = strings.TrimSpace(line) From 7747288a6acbc90337bda2f43e66eac400d1f8d5 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Wed, 15 May 2024 14:45:22 -0400 Subject: [PATCH 04/15] Adds parsing for ini file Goal is to have values stored in ini file which is now parsed as expected --- pkg/trace/go.mod | 1 + pkg/trace/go.sum | 2 ++ pkg/trace/stats/concentrator.go | 26 ++++++++++++++++++++------ pkg/trace/stats/peer_tags.csv | 1 - pkg/trace/stats/peer_tags.ini | 22 ++++++++++++++++++++++ 5 files changed, 45 insertions(+), 7 deletions(-) delete mode 100644 pkg/trace/stats/peer_tags.csv create mode 100644 pkg/trace/stats/peer_tags.ini diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index be5da7ec84328..d831458e6e375 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -45,6 +45,7 @@ require ( golang.org/x/time v0.5.0 google.golang.org/grpc v1.62.1 google.golang.org/protobuf v1.33.0 + gopkg.in/ini.v1 v1.67.0 k8s.io/apimachinery v0.25.5 ) diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index cf5cef10fab0d..5def4faa8ab6f 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -431,6 +431,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index 0c6c10a5609a8..78dca5f680bfe 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -12,6 +12,8 @@ import ( "sync" "time" + "gopkg.in/ini.v1" + pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/trace/config" "github.com/DataDog/datadog-agent/pkg/trace/log" @@ -56,15 +58,27 @@ type Concentrator struct { statsd statsd.ClientInterface } -//go:embed peer_tags.csv -var peerTagFile string +//go:embed peer_tags.ini +var peerTagFile []byte var defaultPeerTags = func() []string { - lines := strings.Split(strings.TrimSpace(peerTagFile), "\n") - tags := make([]string, len(lines)) - for i, line := range lines { - tags[i] = strings.TrimSpace(line) + cfg, _ := ini.Load(peerTagFile) + keys := cfg.Section("dd.apm.peer.tags").Keys() + + var tags []string = []string{"_dd.base_service", "cassandra.cluster", "hazelcast.instance", "statemachinename"} + if len(keys) > 0 { + for _, key := range keys { + value := strings.Split(key.Value(), ",") + for _, tag := range value { + if tag != "grpc.host" && tag != "http.host" { + tags = append(tags, tag) + } + } + } } + + sort.Strings(tags) + return tags }() diff --git a/pkg/trace/stats/peer_tags.csv b/pkg/trace/stats/peer_tags.csv deleted file mode 100644 index 28782949e6519..0000000000000 --- a/pkg/trace/stats/peer_tags.csv +++ /dev/null @@ -1 +0,0 @@ -"_dd.base_service","amqp.destination","amqp.exchange","amqp.queue","aws.queue.name","bucketname","cassandra.cluster","db.cassandra.contact.points","db.couchbase.seed.nodes","db.hostname","db.instance","db.name","db.system","hazelcast.instance","messaging.kafka.bootstrap.servers","mongodb.db","msmq.queue.path","net.peer.name","network.destination.name","peer.hostname","peer.service","queuename","rpc.service","rulename","server.address","statemachinename","streamname","tablename","topicname" diff --git a/pkg/trace/stats/peer_tags.ini b/pkg/trace/stats/peer_tags.ini new file mode 100644 index 0000000000000..3f0cc291435e0 --- /dev/null +++ b/pkg/trace/stats/peer_tags.ini @@ -0,0 +1,22 @@ +# if a peer entity tag is expected to be set at the tracer, and we want to keep it, then it must be included +# in the list. I.e., since we have at least one tracer which actually sets `peer.hostname` directly, we need +# to set `peer.hostname = "peer.hostname" else it will get stripped out +# the order of the precursors matters, as it is "first one wins". so if we expect some spans to have more than one +# precursor, and the precursors may not always have the exact same meaning, then put the higher priority one at the front +[dd.apm.peer.tags] +peer.hostname = "peer.hostname,hostname,net.peer.name,db.hostname,network.destination.name,grpc.host,http.host,server.address" +peer.cassandra.contact.points = "db.cassandra.contact.points" +peer.couchbase.seed.nodes = "db.couchbase.seed.nodes" +peer.rpc.service = "rpc.service" +peer.rpc.system = "rpc.system" +peer.aws.s3.bucket = "bucketname" +peer.aws.sqs.queue = "queuename" +peer.aws.kinesis.stream = "streamname" +peer.aws.dynamodb.table = "tablename" +peer.db.system = "db.system" +peer.db.name = "db.name,mongodb.db,db.instance,cassandra.keyspace" +peer.messaging.system = "messaging.system" +peer.messaging.destination = "topicname,messaging.destination,messaging.destination.name,messaging.rabbitmq.exchange,amqp.destination,amqp.queue,amqp.exchange,msmq.queue.path,aws.queue.name" +peer.kafka.bootstrap.servers = "messaging.kafka.bootstrap.servers" +peer.cluster.name = "peer.cluster.name" +peer.service = "peer.service" From 57f83720c9bd8476c0858e610939fae8e60f7943 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Fri, 17 May 2024 16:13:27 -0400 Subject: [PATCH 05/15] Adds conditions to ensure tag list matches current behavior Due to mismatches between remapping on backend and previous list, several tags are inserted and removed to ensure the final list is the same as before This may change in the future in order to ensure all behavior is consistent Tests are also added to ensure similar behavior --- pkg/trace/stats/concentrator.go | 11 ++++++-- pkg/trace/stats/concentrator_test.go | 41 ++++++++++++++++++++++++++++ pkg/trace/stats/peer_tags.ini | 22 +++++++-------- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index 78dca5f680bfe..74563771bc2d1 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -65,12 +65,19 @@ var defaultPeerTags = func() []string { cfg, _ := ini.Load(peerTagFile) keys := cfg.Section("dd.apm.peer.tags").Keys() - var tags []string = []string{"_dd.base_service", "cassandra.cluster", "hazelcast.instance", "statemachinename"} + var tags []string = []string{"_dd.base_service", "cassandra.cluster", + "hazelcast.instance", "rulename", "statemachinename"} + if len(keys) > 0 { for _, key := range keys { value := strings.Split(key.Value(), ",") for _, tag := range value { - if tag != "grpc.host" && tag != "http.host" { + if tag != "grpc.host" && tag != "http.host" && + tag != "cassandra.keyspace" && tag != "rpc.system" && + tag != "messaging.system" && tag != "peer.cluster.name" && + tag != "hostname" && tag != "out.host" && + tag != "messaging.destination" && tag != "messaging.destination.name" && + tag != "messaging.rabbitmq.exchange" { tags = append(tags, tag) } } diff --git a/pkg/trace/stats/concentrator_test.go b/pkg/trace/stats/concentrator_test.go index f1eb2017bd4e4..8e7131078b569 100644 --- a/pkg/trace/stats/concentrator_test.go +++ b/pkg/trace/stats/concentrator_test.go @@ -1037,3 +1037,44 @@ func TestPreparePeerTags(t *testing.T) { assert.Equal(t, tc.output, preparePeerTags(tc.input...)) } } + +func TestDefaultPeerTags(t *testing.T) { + expectedListOfPeerTags := []string{ + "_dd.base_service", + "amqp.destination", + "amqp.exchange", + "amqp.queue", + "aws.queue.name", + "bucketname", + "cassandra.cluster", + "db.cassandra.contact.points", + "db.couchbase.seed.nodes", + "db.hostname", + "db.instance", + "db.name", + "db.system", + "hazelcast.instance", + "messaging.kafka.bootstrap.servers", + "mongodb.db", + "msmq.queue.path", + "net.peer.name", + "network.destination.name", + "peer.hostname", + "peer.service", + "queuename", + "rpc.service", + "rulename", + "server.address", + "statemachinename", + "streamname", + "tablename", + "topicname", + } + actualListOfPeerTags := defaultPeerTags + + // Sort both arrays for comparison + sort.Strings(actualListOfPeerTags) + sort.Strings(expectedListOfPeerTags) + + assert.Equal(t, expectedListOfPeerTags, actualListOfPeerTags) +} diff --git a/pkg/trace/stats/peer_tags.ini b/pkg/trace/stats/peer_tags.ini index 3f0cc291435e0..1c226912461de 100644 --- a/pkg/trace/stats/peer_tags.ini +++ b/pkg/trace/stats/peer_tags.ini @@ -4,19 +4,19 @@ # the order of the precursors matters, as it is "first one wins". so if we expect some spans to have more than one # precursor, and the precursors may not always have the exact same meaning, then put the higher priority one at the front [dd.apm.peer.tags] -peer.hostname = "peer.hostname,hostname,net.peer.name,db.hostname,network.destination.name,grpc.host,http.host,server.address" -peer.cassandra.contact.points = "db.cassandra.contact.points" -peer.couchbase.seed.nodes = "db.couchbase.seed.nodes" -peer.rpc.service = "rpc.service" -peer.rpc.system = "rpc.system" +peer.aws.dynamodb.table = "tablename" +peer.aws.kinesis.stream = "streamname" peer.aws.s3.bucket = "bucketname" peer.aws.sqs.queue = "queuename" -peer.aws.kinesis.stream = "streamname" -peer.aws.dynamodb.table = "tablename" -peer.db.system = "db.system" +peer.cassandra.contact.points = "db.cassandra.contact.points" +peer.cluster.name = "peer.cluster.name" +peer.couchbase.seed.nodes = "db.couchbase.seed.nodes" peer.db.name = "db.name,mongodb.db,db.instance,cassandra.keyspace" -peer.messaging.system = "messaging.system" -peer.messaging.destination = "topicname,messaging.destination,messaging.destination.name,messaging.rabbitmq.exchange,amqp.destination,amqp.queue,amqp.exchange,msmq.queue.path,aws.queue.name" +peer.db.system = "db.system" +peer.hostname = "peer.hostname,hostname,net.peer.name,db.hostname,network.destination.name,grpc.host,http.host,server.address,out.host" peer.kafka.bootstrap.servers = "messaging.kafka.bootstrap.servers" -peer.cluster.name = "peer.cluster.name" +peer.messaging.destination = "topicname,messaging.destination,messaging.destination.name,messaging.rabbitmq.exchange,amqp.destination,amqp.queue,amqp.exchange,msmq.queue.path,aws.queue.name" +peer.messaging.system = "messaging.system" +peer.rpc.service = "rpc.service" +peer.rpc.system = "rpc.system" peer.service = "peer.service" From c9addb2e37ac3e6a2987e189a0109f999bf4cea3 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Wed, 22 May 2024 10:32:19 -0400 Subject: [PATCH 06/15] Updates generated file to prevent mismatch between aggregation and backend remapping After consulting stats team, new INI file is used to generate tags to aggregate on --- pkg/trace/go.mod | 2 +- pkg/trace/stats/concentrator.go | 12 ++---------- pkg/trace/stats/concentrator_test.go | 13 +++++++++---- pkg/trace/stats/peer_tags.ini | 3 +-- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index 1f201e62c5b47..bb9b39ec4081c 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -46,7 +46,7 @@ require ( google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.34.0 gopkg.in/ini.v1 v1.67.0 - k8s.io/apimachinery v0.25.5 + k8s.io/apimachinery v0.25.5 ) require ( diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index 74563771bc2d1..a3daf8ab295ef 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -65,21 +65,13 @@ var defaultPeerTags = func() []string { cfg, _ := ini.Load(peerTagFile) keys := cfg.Section("dd.apm.peer.tags").Keys() - var tags []string = []string{"_dd.base_service", "cassandra.cluster", - "hazelcast.instance", "rulename", "statemachinename"} + var tags []string = []string{"_dd.base_service"} if len(keys) > 0 { for _, key := range keys { value := strings.Split(key.Value(), ",") for _, tag := range value { - if tag != "grpc.host" && tag != "http.host" && - tag != "cassandra.keyspace" && tag != "rpc.system" && - tag != "messaging.system" && tag != "peer.cluster.name" && - tag != "hostname" && tag != "out.host" && - tag != "messaging.destination" && tag != "messaging.destination.name" && - tag != "messaging.rabbitmq.exchange" { - tags = append(tags, tag) - } + tags = append(tags, tag) } } } diff --git a/pkg/trace/stats/concentrator_test.go b/pkg/trace/stats/concentrator_test.go index 8e7131078b569..be1706b208053 100644 --- a/pkg/trace/stats/concentrator_test.go +++ b/pkg/trace/stats/concentrator_test.go @@ -1046,15 +1046,21 @@ func TestDefaultPeerTags(t *testing.T) { "amqp.queue", "aws.queue.name", "bucketname", - "cassandra.cluster", + "cassandra.keyspace", "db.cassandra.contact.points", "db.couchbase.seed.nodes", "db.hostname", "db.instance", "db.name", "db.system", - "hazelcast.instance", + "grpc.host", + "hostname", + "http.host", + "messaging.destination", + "messaging.destination.name", "messaging.kafka.bootstrap.servers", + "messaging.rabbitmq.exchange", + "messaging.system", "mongodb.db", "msmq.queue.path", "net.peer.name", @@ -1063,9 +1069,8 @@ func TestDefaultPeerTags(t *testing.T) { "peer.service", "queuename", "rpc.service", - "rulename", + "rpc.system", "server.address", - "statemachinename", "streamname", "tablename", "topicname", diff --git a/pkg/trace/stats/peer_tags.ini b/pkg/trace/stats/peer_tags.ini index 1c226912461de..5fe93dabd6956 100644 --- a/pkg/trace/stats/peer_tags.ini +++ b/pkg/trace/stats/peer_tags.ini @@ -9,11 +9,10 @@ peer.aws.kinesis.stream = "streamname" peer.aws.s3.bucket = "bucketname" peer.aws.sqs.queue = "queuename" peer.cassandra.contact.points = "db.cassandra.contact.points" -peer.cluster.name = "peer.cluster.name" peer.couchbase.seed.nodes = "db.couchbase.seed.nodes" peer.db.name = "db.name,mongodb.db,db.instance,cassandra.keyspace" peer.db.system = "db.system" -peer.hostname = "peer.hostname,hostname,net.peer.name,db.hostname,network.destination.name,grpc.host,http.host,server.address,out.host" +peer.hostname = "peer.hostname,hostname,net.peer.name,db.hostname,network.destination.name,grpc.host,http.host,server.address" peer.kafka.bootstrap.servers = "messaging.kafka.bootstrap.servers" peer.messaging.destination = "topicname,messaging.destination,messaging.destination.name,messaging.rabbitmq.exchange,amqp.destination,amqp.queue,amqp.exchange,msmq.queue.path,aws.queue.name" peer.messaging.system = "messaging.system" From 38885a481601ec4c0c9dd6c5fdfb326e26754090 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Wed, 22 May 2024 10:38:57 -0400 Subject: [PATCH 07/15] Update go.mod spacing issue on go.mod --- pkg/trace/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index bb9b39ec4081c..b1c690ec3f6de 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -46,7 +46,7 @@ require ( google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.34.0 gopkg.in/ini.v1 v1.67.0 - k8s.io/apimachinery v0.25.5 + k8s.io/apimachinery v0.25.5 ) require ( From 1f834b1976a19465bd2e553a293d7d0ebee7cab1 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Thu, 23 May 2024 10:28:52 -0400 Subject: [PATCH 08/15] Updates list to include new otel peer source tags --- pkg/trace/stats/concentrator.go | 4 +--- pkg/trace/stats/concentrator_test.go | 3 +++ pkg/trace/stats/peer_tags.ini | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index a3daf8ab295ef..7e56464de18dc 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -70,9 +70,7 @@ var defaultPeerTags = func() []string { if len(keys) > 0 { for _, key := range keys { value := strings.Split(key.Value(), ",") - for _, tag := range value { - tags = append(tags, tag) - } + tags = append(tags, value...) } } diff --git a/pkg/trace/stats/concentrator_test.go b/pkg/trace/stats/concentrator_test.go index be1706b208053..1903c57c20bed 100644 --- a/pkg/trace/stats/concentrator_test.go +++ b/pkg/trace/stats/concentrator_test.go @@ -1045,6 +1045,7 @@ func TestDefaultPeerTags(t *testing.T) { "amqp.exchange", "amqp.queue", "aws.queue.name", + "aws.s3.bucket", "bucketname", "cassandra.keyspace", "db.cassandra.contact.points", @@ -1052,10 +1053,12 @@ func TestDefaultPeerTags(t *testing.T) { "db.hostname", "db.instance", "db.name", + "db.namespace", "db.system", "grpc.host", "hostname", "http.host", + "http.server_name", "messaging.destination", "messaging.destination.name", "messaging.kafka.bootstrap.servers", diff --git a/pkg/trace/stats/peer_tags.ini b/pkg/trace/stats/peer_tags.ini index 5fe93dabd6956..d1b36aacf7e5c 100644 --- a/pkg/trace/stats/peer_tags.ini +++ b/pkg/trace/stats/peer_tags.ini @@ -6,13 +6,13 @@ [dd.apm.peer.tags] peer.aws.dynamodb.table = "tablename" peer.aws.kinesis.stream = "streamname" -peer.aws.s3.bucket = "bucketname" +peer.aws.s3.bucket = "bucketname,aws.s3.bucket" peer.aws.sqs.queue = "queuename" peer.cassandra.contact.points = "db.cassandra.contact.points" peer.couchbase.seed.nodes = "db.couchbase.seed.nodes" -peer.db.name = "db.name,mongodb.db,db.instance,cassandra.keyspace" +peer.db.name = "db.name,mongodb.db,db.instance,cassandra.keyspace,db.namespace" peer.db.system = "db.system" -peer.hostname = "peer.hostname,hostname,net.peer.name,db.hostname,network.destination.name,grpc.host,http.host,server.address" +peer.hostname = "peer.hostname,hostname,net.peer.name,db.hostname,network.destination.name,grpc.host,http.host,server.address,http.server_name" peer.kafka.bootstrap.servers = "messaging.kafka.bootstrap.servers" peer.messaging.destination = "topicname,messaging.destination,messaging.destination.name,messaging.rabbitmq.exchange,amqp.destination,amqp.queue,amqp.exchange,msmq.queue.path,aws.queue.name" peer.messaging.system = "messaging.system" From 4512bd6332593547ab09ace5e7649fe816c233d2 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Tue, 28 May 2024 11:57:39 -0400 Subject: [PATCH 09/15] removes unnecessary length check --- pkg/trace/stats/concentrator.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index 7e56464de18dc..777ec67e5b502 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -67,11 +67,9 @@ var defaultPeerTags = func() []string { var tags []string = []string{"_dd.base_service"} - if len(keys) > 0 { - for _, key := range keys { - value := strings.Split(key.Value(), ",") - tags = append(tags, value...) - } + for _, key := range keys { + value := strings.Split(key.Value(), ",") + tags = append(tags, value...) } sort.Strings(tags) From 6803c881cf72713fbe4ba5e79cd58cf9fd53be76 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Tue, 28 May 2024 14:13:40 -0400 Subject: [PATCH 10/15] Added error handling for parsing INI file Synced go.mod files --- .../otlp/components/pipeline/provider/go.mod | 24 ------------------- .../otlp/components/statsprocessor/go.mod | 1 + .../otlp/components/statsprocessor/go.sum | 2 ++ pkg/trace/stats/concentrator.go | 10 +++++--- 4 files changed, 10 insertions(+), 27 deletions(-) diff --git a/comp/otelcol/otlp/components/pipeline/provider/go.mod b/comp/otelcol/otlp/components/pipeline/provider/go.mod index 12a7599428fd3..b8e737ffcc0ff 100644 --- a/comp/otelcol/otlp/components/pipeline/provider/go.mod +++ b/comp/otelcol/otlp/components/pipeline/provider/go.mod @@ -190,30 +190,6 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector v0.100.0 // indirect - go.opentelemetry.io/collector/config/configtelemetry v0.100.0 // indirect - go.opentelemetry.io/collector/consumer v0.100.0 // indirect - go.opentelemetry.io/collector/featuregate v1.6.0 // indirect - go.opentelemetry.io/collector/pdata v1.6.0 // indirect - go.opentelemetry.io/collector/pdata/testdata v0.100.0 // indirect - go.opentelemetry.io/collector/semconv v0.100.0 // indirect - go.opentelemetry.io/collector/service v0.100.0 // indirect - go.opentelemetry.io/contrib/config v0.5.0 // indirect - go.opentelemetry.io/contrib/propagators/b3 v1.25.0 // indirect - go.opentelemetry.io/otel v1.25.0 // indirect - go.opentelemetry.io/otel/bridge/opencensus v1.25.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.25.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.25.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.25.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.25.0 // indirect - go.opentelemetry.io/otel/exporters/prometheus v0.47.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.25.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.25.0 // indirect - go.opentelemetry.io/otel/metric v1.25.0 // indirect - go.opentelemetry.io/otel/sdk v1.25.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.25.0 // indirect - go.opentelemetry.io/otel/trace v1.25.0 // indirect - go.opentelemetry.io/collector/component v0.100.0 // indirect go.opentelemetry.io/collector/config/configauth v0.100.0 // indirect go.opentelemetry.io/collector/config/configcompression v1.7.0 // indirect go.opentelemetry.io/collector/config/confighttp v0.100.0 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index f79c5e7056aab..2769abb4681b4 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -101,6 +101,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 // indirect google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index ff32884e6938c..0ef2955aec8e3 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -286,6 +286,8 @@ google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/trace/stats/concentrator.go b/pkg/trace/stats/concentrator.go index 777ec67e5b502..03c782bdbc7d1 100644 --- a/pkg/trace/stats/concentrator.go +++ b/pkg/trace/stats/concentrator.go @@ -62,11 +62,15 @@ type Concentrator struct { var peerTagFile []byte var defaultPeerTags = func() []string { - cfg, _ := ini.Load(peerTagFile) - keys := cfg.Section("dd.apm.peer.tags").Keys() - var tags []string = []string{"_dd.base_service"} + cfg, err := ini.Load(peerTagFile) + if err != nil { + log.Error("Error loading file for peer tags: ", err) + return tags + } + keys := cfg.Section("dd.apm.peer.tags").Keys() + for _, key := range keys { value := strings.Split(key.Value(), ",") tags = append(tags, value...) From 65275d8e61aa4c657cd4b27ba2c7e1efa277c8a8 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Wed, 29 May 2024 10:49:23 -0400 Subject: [PATCH 11/15] fix go.mod files --- pkg/trace/stats/oteltest/go.mod | 1 + pkg/trace/stats/oteltest/go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 6a3c378f83f54..f2e86350ceeb3 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -90,6 +90,7 @@ require ( golang.org/x/tools v0.16.1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index c41384764df4a..37b2159b19d76 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -288,6 +288,8 @@ google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHh gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From dfd18ef097f006c0a3b95446cb82f50eb1ce408c Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Wed, 29 May 2024 15:41:10 -0400 Subject: [PATCH 12/15] add serverless dependencies --- cmd/serverless/dependencies_linux_amd64.txt | 4 +++- cmd/serverless/dependencies_linux_arm64.txt | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt index 8589b5f124db7..632c05a75a783 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/dependencies_linux_amd64.txt @@ -907,6 +907,7 @@ gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof gopkg.in/DataDog/dd-trace-go.v1/internal/version +ggopkg.in/ini.v1 gopkg.in/yaml.v2 gopkg.in/yaml.v3 hash @@ -918,6 +919,7 @@ html/template internal/abi internal/bisect internal/bytealg +internal/chacha8rand internal/coverage/rtcov internal/cpu internal/fmtsort @@ -1022,4 +1024,4 @@ vendor/golang.org/x/sys/cpu vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm \ No newline at end of file +vendor/golang.org/x/text/unicode/norm diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt index 518a1dc9e0135..5394039044c41 100644 --- a/cmd/serverless/dependencies_linux_arm64.txt +++ b/cmd/serverless/dependencies_linux_arm64.txt @@ -906,6 +906,7 @@ gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof gopkg.in/DataDog/dd-trace-go.v1/internal/version +gopkg.in/ini.v1 gopkg.in/yaml.v2 gopkg.in/yaml.v3 hash @@ -917,6 +918,7 @@ html/template internal/abi internal/bisect internal/bytealg +internal/chacha8rand internal/coverage/rtcov internal/cpu internal/fmtsort From 48fcf342d88fd9c01c7fcf90e5bf35c8b9448975 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Wed, 29 May 2024 16:08:24 -0400 Subject: [PATCH 13/15] fix serverless dep typo --- cmd/serverless/dependencies_linux_amd64.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt index 147fa8b82e8a7..d1f3b90ad2a4c 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/dependencies_linux_amd64.txt @@ -908,7 +908,7 @@ gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames gopkg.in/DataDog/dd-trace-go.v1/internal/telemetry gopkg.in/DataDog/dd-trace-go.v1/internal/traceprof gopkg.in/DataDog/dd-trace-go.v1/internal/version -ggopkg.in/ini.v1 +gopkg.in/ini.v1 gopkg.in/yaml.v2 gopkg.in/yaml.v3 hash @@ -1025,4 +1025,4 @@ vendor/golang.org/x/sys/cpu vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm +vendor/golang.org/x/text/unicode/norm \ No newline at end of file From 6907b70b92d268f8bdb5157fab71567ff57d3169 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Thu, 30 May 2024 08:45:49 -0400 Subject: [PATCH 14/15] fix deps test --- cmd/serverless/dependencies_linux_amd64.txt | 3 +-- cmd/serverless/dependencies_linux_arm64.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt index d1f3b90ad2a4c..f864da77cf124 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/dependencies_linux_amd64.txt @@ -920,7 +920,6 @@ html/template internal/abi internal/bisect internal/bytealg -internal/chacha8rand internal/coverage/rtcov internal/cpu internal/fmtsort @@ -1025,4 +1024,4 @@ vendor/golang.org/x/sys/cpu vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm \ No newline at end of file +vendor/golang.org/x/text/unicode/norm diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt index b04fb61a1613b..15a27491dbf30 100644 --- a/cmd/serverless/dependencies_linux_arm64.txt +++ b/cmd/serverless/dependencies_linux_arm64.txt @@ -919,7 +919,6 @@ html/template internal/abi internal/bisect internal/bytealg -internal/chacha8rand internal/coverage/rtcov internal/cpu internal/fmtsort @@ -1023,4 +1022,4 @@ vendor/golang.org/x/net/idna vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm \ No newline at end of file +vendor/golang.org/x/text/unicode/norm From 1f5cc9f420223311c1bdec7f377ee2d1d3e018b1 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Thu, 30 May 2024 11:47:03 -0400 Subject: [PATCH 15/15] remove newline at end of dep file --- cmd/serverless/dependencies_linux_amd64.txt | 2 +- cmd/serverless/dependencies_linux_arm64.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt index 93cb740aa6ba5..7f9dfe9bfa5ec 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/dependencies_linux_amd64.txt @@ -1025,4 +1025,4 @@ vendor/golang.org/x/sys/cpu vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm +vendor/golang.org/x/text/unicode/norm \ No newline at end of file diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt index 89d9c2e570578..e91c33722e402 100644 --- a/cmd/serverless/dependencies_linux_arm64.txt +++ b/cmd/serverless/dependencies_linux_arm64.txt @@ -1023,4 +1023,4 @@ vendor/golang.org/x/net/idna vendor/golang.org/x/text/secure/bidirule vendor/golang.org/x/text/transform vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm +vendor/golang.org/x/text/unicode/norm \ No newline at end of file