diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ec5ac92f6a2e5..50f8ff18a0c96 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -176,6 +176,7 @@ /.gitlab/package_build/ @DataDog/agent-delivery /.gitlab/package_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/package_build/installer.yml @DataDog/agent-delivery @DataDog/fleet /.gitlab/packaging/ @DataDog/agent-delivery /.gitlab/benchmarks/benchmarks.yml @DataDog/agent-apm diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index 9008ad3e857b1..944343f20da2f 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -1,7 +1,4 @@ --- variables: - # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev - # and check the job creating the image to make sure you have the right SHA prefix - TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" - # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 6459608ed9fa + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 221bbc806266 + TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: '' diff --git a/Dockerfiles/agent-ot/Dockerfile.agent-otel b/Dockerfiles/agent-ot/Dockerfile.agent-otel index 78534598a581c..49ea309af7019 100644 --- a/Dockerfiles/agent-ot/Dockerfile.agent-otel +++ b/Dockerfiles/agent-ot/Dockerfile.agent-otel @@ -1,5 +1,5 @@ -ARG AGENT_VERSION=7.57.0-v1.0-ot-beta-jmx -ARG AGENT_BRANCH=7.57.x-otel-beta-v1 +ARG AGENT_VERSION=7.59.0-v1.1.0-ot-beta-jmx +ARG AGENT_BRANCH=7.59.x # Use the Ubuntu Slim AMD64 base image FROM ubuntu:24.04 AS builder diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 92e5c072e3c8e..13955c7e8a940 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2906,9 +2906,6 @@ core,google.golang.org/protobuf/types/known/timestamppb,BSD-3-Clause,Copyright ( core,google.golang.org/protobuf/types/known/wrapperspb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,google.golang.org/protobuf/types/pluginpb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,gopkg.in/DataDog/dd-trace-go.v1/appsec/events,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/datastreams/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext,Apache-2.0,"Copyright 2016-Present Datadog, Inc." diff --git a/cmd/installer-downloader/main.go b/cmd/installer-downloader/main.go index f7b67910ae94e..60e2141af944e 100644 --- a/cmd/installer-downloader/main.go +++ b/cmd/installer-downloader/main.go @@ -44,12 +44,12 @@ func main() { ctx := context.Background() t := telemetry.NewTelemetry(env.HTTPClient(), env.APIKey, env.Site, fmt.Sprintf("datadog-installer-downloader-%s", Flavor)) - _ = t.Start(ctx) - defer func() { _ = t.Stop(ctx) }() var err error span, ctx := telemetry.StartSpanFromEnv(ctx, fmt.Sprintf("downloader-%s", Flavor)) - defer func() { span.Finish(err) }() err = runDownloader(ctx, env, Version, Flavor) + + span.Finish(err) + t.Stop() if err != nil { fmt.Fprintf(os.Stderr, "Installation failed: %v\n", err) os.Exit(1) diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 66ec6b1c74c57..f394d9955d407 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -87,7 +87,7 @@ func UnprivilegedCommands(_ *command.GlobalParams) []*cobra.Command { type cmd struct { t *telemetry.Telemetry ctx context.Context - span telemetry.Span + span *telemetry.Span env *env.Env } @@ -107,10 +107,7 @@ func newCmd(operation string) *cmd { func (c *cmd) Stop(err error) { c.span.Finish(err) if c.t != nil { - err := c.t.Stop(context.Background()) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to stop telemetry: %v\n", err) - } + c.t.Stop() } } @@ -225,11 +222,6 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry { site = config.Site } t := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands - err := t.Start(context.Background()) - if err != nil { - fmt.Printf("failed to start telemetry: %v\n", err) - return nil - } return t } diff --git a/cmd/installer/subcommands/installer/umask_nix.go b/cmd/installer/subcommands/installer/umask_nix.go index 1fd44c01ac405..dec64eccd6860 100644 --- a/cmd/installer/subcommands/installer/umask_nix.go +++ b/cmd/installer/subcommands/installer/umask_nix.go @@ -14,7 +14,7 @@ import ( ) // setInstallerUmask sets umask 0 to override any inherited umask -func setInstallerUmask(span telemetry.Span) { +func setInstallerUmask(span *telemetry.Span) { oldmask := syscall.Umask(0) span.SetTag("inherited_umask", oldmask) } diff --git a/cmd/installer/subcommands/installer/umask_windows.go b/cmd/installer/subcommands/installer/umask_windows.go index 1b076f92bc389..d8661700cd56e 100644 --- a/cmd/installer/subcommands/installer/umask_windows.go +++ b/cmd/installer/subcommands/installer/umask_windows.go @@ -10,4 +10,4 @@ package installer import "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" // setInstallerUmask no-op on Windows -func setInstallerUmask(_ telemetry.Span) {} +func setInstallerUmask(_ *telemetry.Span) {} diff --git a/cmd/otel-agent/subcommands/run/command.go b/cmd/otel-agent/subcommands/run/command.go index a414674620e44..08c1472998157 100644 --- a/cmd/otel-agent/subcommands/run/command.go +++ b/cmd/otel-agent/subcommands/run/command.go @@ -25,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/core/hostname/remotehostnameimpl" log "github.com/DataDog/datadog-agent/comp/core/log/def" + logfx "github.com/DataDog/datadog-agent/comp/core/log/fx" logtracefx "github.com/DataDog/datadog-agent/comp/core/log/fx-trace" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" @@ -106,8 +107,20 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, fx.Provide(func() coreconfig.Component { return acfg }), + fx.Provide(func(_ coreconfig.Component) log.Params { + return log.ForDaemon(params.LoggerName, "log_file", pkgconfigsetup.DefaultOTelAgentLogFile) + }), + logfx.Module(), + fetchonlyimpl.Module(), + // TODO: don't rely on this pattern; remove this `OptionalModuleWithParams` thing + // and instead adapt OptionalModule to allow parameter passing naturally. + // See: https://github.com/DataDog/datadog-agent/pull/28386 + configsyncimpl.OptionalModuleWithParams(), + fx.Provide(func() configsyncimpl.Params { + return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true) + }), converterfx.Module(), - fx.Provide(func(cp converter.Component) confmap.Converter { + fx.Provide(func(cp converter.Component, _ optional.Option[configsync.Component]) confmap.Converter { return cp }), collectorcontribFx.Module(), diff --git a/cmd/trace-agent/config/remote/config.go b/cmd/trace-agent/config/remote/config.go index d2c511a4586e7..6b650364743a7 100644 --- a/cmd/trace-agent/config/remote/config.go +++ b/cmd/trace-agent/config/remote/config.go @@ -43,7 +43,7 @@ func putBuffer(buffer *bytes.Buffer) { // ConfigHandler is the HTTP handler for configs func ConfigHandler(r *api.HTTPReceiver, cf rcclient.ConfigFetcher, cfg *config.AgentConfig, statsd statsd.ClientInterface, timing timing.Reporter) http.Handler { - cidProvider := api.NewIDProvider(cfg.ContainerProcRoot) + cidProvider := api.NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo) return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer timing.Since("datadog.trace_agent.receiver.config_process_ms", time.Now()) tags := r.TagStats(api.V07, req.Header, "").AsTags() diff --git a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go index d8631536c3a44..6098dcf98f629 100644 --- a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go +++ b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go @@ -168,11 +168,22 @@ func (wcd *AgentCrashDetect) Run() error { } log.Infof("Sending crash: %v", formatText(crash)) - lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), wcd.tconfig.TelemetryConfig.Endpoints, "ddnpm", true) + lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), toTelemEndpoints(wcd.tconfig.TelemetryConfig.Endpoints), "ddnpm", true) lts.SendLog("WARN", formatText(crash)) return nil } +func toTelemEndpoints(endpoints []*traceconfig.Endpoint) []*internaltelemetry.Endpoint { + telemEndpoints := make([]*internaltelemetry.Endpoint, 0, len(endpoints)) + for _, e := range endpoints { + telemEndpoints = append(telemEndpoints, &internaltelemetry.Endpoint{ + Host: e.Host, + APIKey: e.APIKey, + }) + } + return telemEndpoints +} + func newAgentCrashComponent(deps dependencies) agentcrashdetect.Component { instance := &agentCrashComponent{} instance.tconfig = deps.TConfig.Object() diff --git a/comp/core/tagger/collectors/pod_tag_extractor.go b/comp/core/tagger/collectors/pod_tag_extractor.go index 669a71584f609..aded502edaf4c 100644 --- a/comp/core/tagger/collectors/pod_tag_extractor.go +++ b/comp/core/tagger/collectors/pod_tag_extractor.go @@ -34,6 +34,8 @@ func (p *PodTagExtractor) Extract(podEntity *workloadmeta.KubernetesPod, cardina return append(tagInfos.LowCardTags, tagInfos.OrchestratorCardTags...) case types.LowCardinality: return tagInfos.LowCardTags + case types.NoneCardinality: + return []string{} default: log.Errorf("unsupported tag cardinality %v", cardinality) return []string{} diff --git a/comp/core/tagger/impl/tagger.go b/comp/core/tagger/impl/tagger.go index 7023106b1c118..de05b9f6eaa09 100644 --- a/comp/core/tagger/impl/tagger.go +++ b/comp/core/tagger/impl/tagger.go @@ -422,7 +422,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert // | none | empty || empty | // | empty | not empty || container prefix + originFromMsg | // | none | not empty || container prefix + originFromMsg | - if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == "none" { + if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" @@ -460,8 +460,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert } default: // Disable origin detection if cardinality is none - // TODO: The `none` cardinality should be directly supported by the Tagger. - if originInfo.Cardinality == "none" { + if originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" diff --git a/comp/core/tagger/subscriber/subscription_manager_test.go b/comp/core/tagger/subscriber/subscription_manager_test.go index b9684af30e2d4..dbebd8ed23d98 100644 --- a/comp/core/tagger/subscriber/subscription_manager_test.go +++ b/comp/core/tagger/subscriber/subscription_manager_test.go @@ -109,6 +109,21 @@ func TestSubscriptionManager(t *testing.T) { highCardSubscription.Unsubscribe() + // None Cardinality Subscriber + noneCardSubID := "none-card-sub" + noneCardSubscription, err := sm.Subscribe(noneCardSubID, types.NewFilterBuilder().Include(types.EntityIDPrefix("foo")).Build(types.NoneCardinality), nil) + require.NoError(t, err) + + sm.Notify([]types.EntityEvent{ + events["added"], + events["modified"], + events["deleted"], + events["added-with-no-id"], + events["added-with-unmatched-prefix"], + }) + + noneCardSubscription.Unsubscribe() + // Verify low cardinality subscriber received events assertReceivedEvents(t, lowCardSubscription.EventsChan(), []types.EntityEvent{ { @@ -192,6 +207,28 @@ func TestSubscriptionManager(t *testing.T) { }, }, }) + + // Verify none cardinality subscriber received events + assertReceivedEvents(t, noneCardSubscription.EventsChan(), []types.EntityEvent{ + { + EventType: types.EventTypeAdded, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeModified, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeDeleted, + Entity: types.Entity{ + ID: entityID, + }, + }, + }) } func assertReceivedEvents(t *testing.T, ch chan []types.EntityEvent, expectedEvents []types.EntityEvent) { diff --git a/comp/core/tagger/tagstore/entity_tags.go b/comp/core/tagger/tagstore/entity_tags.go index 3488a8e345c16..ed62d45849109 100644 --- a/comp/core/tagger/tagstore/entity_tags.go +++ b/comp/core/tagger/tagstore/entity_tags.go @@ -109,12 +109,16 @@ func (e *EntityTagsWithMultipleSources) getStandard() []string { func (e *EntityTagsWithMultipleSources) getHashedTags(cardinality types.TagCardinality) tagset.HashedTags { e.computeCache() - if cardinality == types.HighCardinality { + switch cardinality { + case types.HighCardinality: return e.cachedAll - } else if cardinality == types.OrchestratorCardinality { + case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} + default: + return e.cachedLow } - return e.cachedLow } func (e *EntityTagsWithMultipleSources) computeCache() { @@ -302,6 +306,8 @@ func (e *EntityTagsWithSingleSource) getHashedTags(cardinality types.TagCardinal return e.cachedAll case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} default: return e.cachedLow } diff --git a/comp/core/tagger/tagstore/entity_tags_test.go b/comp/core/tagger/tagstore/entity_tags_test.go index c93d764377aaf..16e2d4b077f86 100644 --- a/comp/core/tagger/tagstore/entity_tags_test.go +++ b/comp/core/tagger/tagstore/entity_tags_test.go @@ -100,6 +100,12 @@ func TestGetHashedTags(t *testing.T) { []string{"l1:v1", "l2:v2", "service:s1", "o1:v1", "o2:v2", "h1:v1", "h2:v2"}, entityTags.getHashedTags(types.HighCardinality).Get(), ) + + assert.Equal( + t, + []string(nil), + entityTags.getHashedTags(types.NoneCardinality).Get(), + ) } func TestTagsForSource(t *testing.T) { diff --git a/comp/core/tagger/tagstore/tagstore_test.go b/comp/core/tagger/tagstore/tagstore_test.go index 4699aa6ad56cb..0d992b682ca68 100644 --- a/comp/core/tagger/tagstore/tagstore_test.go +++ b/comp/core/tagger/tagstore/tagstore_test.go @@ -88,10 +88,12 @@ func (s *StoreTestSuite) TestLookup() { tagsHigh := s.tagstore.Lookup(entityID, types.HighCardinality) tagsOrch := s.tagstore.Lookup(entityID, types.OrchestratorCardinality) tagsLow := s.tagstore.Lookup(entityID, types.LowCardinality) + tagsNone := s.tagstore.Lookup(entityID, types.NoneCardinality) assert.Len(s.T(), tagsHigh, 4) assert.Len(s.T(), tagsLow, 2) assert.Len(s.T(), tagsOrch, 3) + assert.Nil(s.T(), tagsNone) } func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { @@ -118,10 +120,12 @@ func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { tagsLow := s.tagstore.LookupHashedWithEntityStr(entityID, types.LowCardinality) tagsOrch := s.tagstore.LookupHashedWithEntityStr(entityID, types.OrchestratorCardinality) tagsHigh := s.tagstore.LookupHashedWithEntityStr(entityID, types.HighCardinality) + tagsNone := s.tagstore.LookupHashedWithEntityStr(entityID, types.NoneCardinality) assert.ElementsMatch(s.T(), tagsLow.Get(), []string{"low1", "low2"}) assert.ElementsMatch(s.T(), tagsOrch.Get(), []string{"low1", "low2", "orchestrator1"}) assert.ElementsMatch(s.T(), tagsHigh.Get(), []string{"low1", "low2", "orchestrator1", "high1"}) + assert.ElementsMatch(s.T(), tagsNone.Get(), []string{}) } func (s *StoreTestSuite) TestLookupStandard() { diff --git a/comp/core/tagger/telemetry/telemetry.go b/comp/core/tagger/telemetry/telemetry.go index 97c7153d9c052..0682475eb6d8e 100644 --- a/comp/core/tagger/telemetry/telemetry.go +++ b/comp/core/tagger/telemetry/telemetry.go @@ -68,6 +68,7 @@ type Store struct { LowCardinalityQueries CardinalityTelemetry OrchestratorCardinalityQueries CardinalityTelemetry HighCardinalityQueries CardinalityTelemetry + NoneCardinalityQueries CardinalityTelemetry UnknownCardinalityQueries CardinalityTelemetry } @@ -144,6 +145,7 @@ func NewStore(telemetryComp telemetry.Component) *Store { LowCardinalityQueries: newCardinalityTelemetry(queries, types.LowCardinalityString), OrchestratorCardinalityQueries: newCardinalityTelemetry(queries, types.OrchestratorCardinalityString), HighCardinalityQueries: newCardinalityTelemetry(queries, types.HighCardinalityString), + NoneCardinalityQueries: newCardinalityTelemetry(queries, types.NoneCardinalityString), UnknownCardinalityQueries: newCardinalityTelemetry(queries, types.UnknownCardinalityString), } }) @@ -160,6 +162,8 @@ func (s *Store) QueriesByCardinality(card types.TagCardinality) *CardinalityTele return &s.OrchestratorCardinalityQueries case types.HighCardinality: return &s.HighCardinalityQueries + case types.NoneCardinality: + return &s.NoneCardinalityQueries default: return &s.UnknownCardinalityQueries } diff --git a/comp/core/tagger/types/types.go b/comp/core/tagger/types/types.go index e45f9e6daa084..dc53af7baf27b 100644 --- a/comp/core/tagger/types/types.go +++ b/comp/core/tagger/types/types.go @@ -78,6 +78,7 @@ const ( LowCardinality TagCardinality = iota OrchestratorCardinality HighCardinality + NoneCardinality ) // Entity is an entity ID + tags. @@ -92,6 +93,10 @@ type Entity struct { // GetTags flattens all tags from all cardinalities into a single slice of tag // strings. func (e Entity) GetTags(cardinality TagCardinality) []string { + if cardinality == NoneCardinality { + return []string{} + } + tagArrays := make([][]string, 0, 3) tagArrays = append(tagArrays, e.LowCardinalityTags) @@ -117,6 +122,11 @@ func (e Entity) Copy(cardinality TagCardinality) Entity { case LowCardinality: newEntity.HighCardinalityTags = nil newEntity.OrchestratorCardinalityTags = nil + case NoneCardinality: + newEntity.HighCardinalityTags = nil + newEntity.OrchestratorCardinalityTags = nil + newEntity.LowCardinalityTags = nil + newEntity.StandardTags = nil } return newEntity @@ -131,6 +141,8 @@ const ( ShortOrchestratorCardinalityString = "orch" // HighCardinalityString is the string representation of the high cardinality HighCardinalityString = "high" + // NoneCardinalityString is the string representation of the none cardinality + NoneCardinalityString = "none" // UnknownCardinalityString represents an unknown level of cardinality UnknownCardinalityString = "unknown" ) @@ -145,6 +157,8 @@ func StringToTagCardinality(c string) (TagCardinality, error) { return OrchestratorCardinality, nil case LowCardinalityString: return LowCardinality, nil + case NoneCardinalityString: + return NoneCardinality, nil default: return LowCardinality, fmt.Errorf("unsupported value %s received for tag cardinality", c) } @@ -160,6 +174,8 @@ func TagCardinalityToString(c TagCardinality) string { return OrchestratorCardinalityString case LowCardinality: return LowCardinalityString + case NoneCardinality: + return NoneCardinalityString default: return UnknownCardinalityString } diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 7af5d796d92fb..21d3f17b9066a 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -13,6 +13,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/tagger/tags => ../../../core/tagger/tags github.com/DataDog/datadog-agent/comp/core/tagger/types => ../../../core/tagger/types github.com/DataDog/datadog-agent/comp/core/tagger/utils => ../../../core/tagger/utils @@ -151,6 +152,7 @@ require ( require go.opentelemetry.io/collector/extension/extensiontest v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/server.go b/comp/otelcol/ddflareextension/impl/server.go index ce1d2ceadef52..233dcddee1977 100644 --- a/comp/otelcol/ddflareextension/impl/server.go +++ b/comp/otelcol/ddflareextension/impl/server.go @@ -116,7 +116,7 @@ func newServer(endpoint string, handler http.Handler, auth bool) (*server, error // no easy way currently to pass required bearer auth token to OSS collector; // skip the validation if running inside a separate collector // TODO: determine way to allow OSS collector to authenticate with agent, OTEL-2226 - if auth { + if auth && util.GetAuthToken() != "" { r.Use(validateToken) } diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index c3e28a412d2e6..301ef0f3b1f48 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../../../core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../../../../def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../../../../forwarder/defaultforwarder @@ -138,6 +139,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/status v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder v0.56.0-rc.3 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index 4ad268c10dfcb..18606500a04d2 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -3,6 +3,7 @@ module github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsproces go 1.22.0 replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../metricsclient github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../../../../comp/trace/compression/impl-gzip @@ -33,6 +34,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.56.0-rc.3 // indirect diff --git a/comp/trace/config/setup.go b/comp/trace/config/setup.go index f19c5eeaf8a4f..06a157dc6418b 100644 --- a/comp/trace/config/setup.go +++ b/comp/trace/config/setup.go @@ -19,13 +19,14 @@ import ( "go.opentelemetry.io/collector/component/componenttest" - apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" corecompcfg "github.com/DataDog/datadog-agent/comp/core/config" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/configcheck" + apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/env" "github.com/DataDog/datadog-agent/pkg/config/model" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -121,6 +122,9 @@ func prepareConfig(c corecompcfg.Component, tagger tagger.Component) (*config.Ag cfg.ContainerTags = func(cid string) ([]string, error) { return tagger.Tag(types.NewEntityID(types.ContainerID, cid), types.HighCardinality) } + cfg.ContainerIDFromOriginInfo = func(originInfo origindetection.OriginInfo) (string, error) { + return tagger.GenerateContainerIDFromOriginInfo(originInfo) + } cfg.ContainerProcRoot = coreConfigObject.GetString("container_proc_root") cfg.GetAgentAuthToken = apiutil.GetAuthToken return cfg, nil diff --git a/comp/updater/telemetry/telemetryimpl/telemetry.go b/comp/updater/telemetry/telemetryimpl/telemetry.go index d8f961e8398e6..ee1bab4efdd56 100644 --- a/comp/updater/telemetry/telemetryimpl/telemetry.go +++ b/comp/updater/telemetry/telemetryimpl/telemetry.go @@ -7,10 +7,10 @@ package telemetryimpl import ( + "context" "net/http" "go.uber.org/fx" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/updater/telemetry" @@ -38,13 +38,7 @@ func newTelemetry(deps dependencies) (telemetry.Component, error) { client := &http.Client{ Transport: httputils.CreateHTTPTransport(deps.Config), } - telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon", - fleettelemetry.WithSamplingRules( - tracer.NameServiceRule("cdn.*", "datadog-installer-daemon", 0.1), - tracer.NameServiceRule("*garbage_collect*", "datadog-installer-daemon", 0.05), - tracer.NameServiceRule("HTTPClient.*", "datadog-installer-daemon", 0.05), - ), - ) - deps.Lc.Append(fx.Hook{OnStart: telemetry.Start, OnStop: telemetry.Stop}) + telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon") + deps.Lc.Append(fx.Hook{OnStop: func(context.Context) error { telemetry.Stop(); return nil }}) return telemetry, nil } diff --git a/go.mod b/go.mod index 30981dbec9ce2..c8de7db9cc5f0 100644 --- a/go.mod +++ b/go.mod @@ -654,7 +654,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 github.com/DataDog/datadog-agent/comp/core/status v0.59.0-rc.6 github.com/DataDog/datadog-agent/comp/core/status/statusimpl v0.56.0-rc.3 - github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-00010101000000-000000000000 + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/DataDog/datadog-agent/comp/core/tagger/tags v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/core/tagger/types v0.59.0 github.com/DataDog/datadog-agent/comp/core/telemetry v0.59.0 diff --git a/modules.yml b/modules.yml index 291db46bc53cc..5d5e5cb2057ff 100644 --- a/modules.yml +++ b/modules.yml @@ -35,7 +35,8 @@ modules: comp/core/status: used_by_otel: true comp/core/status/statusimpl: default - comp/core/tagger/origindetection: default + comp/core/tagger/origindetection: + used_by_otel: true comp/core/tagger/tags: used_by_otel: true comp/core/tagger/types: diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 2fe37a032fc24..1cd3c096363f2 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -54,11 +54,11 @@ # Don't forward CC and CXX to python extensions Makefile, it's quite unlikely that any non default # compiler we use would end up being available in the system/docker image used by customers - if linux_target? && ENV["CC"] + if linux_target? && env["CC"] command "sed -i \"s/^CC=[[:space:]]*${CC}/CC=gcc/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CC}/gcc/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end - if linux_target? && ENV["CXX"] + if linux_target? && env["CXX"] command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CXX}/g++/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 24f8d81503459..fdb1d427e9d15 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -79,7 +79,7 @@ const ( DefaultRuntimePoliciesDir = "/etc/datadog-agent/runtime-security.d" // DefaultCompressorKind is the default compressor. Options available are 'zlib' and 'zstd' - DefaultCompressorKind = "zlib" + DefaultCompressorKind = "zstd" // DefaultZstdCompressionLevel is the default compression level for `zstd`. // Compression level 1 provides the lowest compression ratio, but uses much less RSS especially diff --git a/pkg/config/setup/system_probe_cws.go b/pkg/config/setup/system_probe_cws.go index d2cc6276907d3..c689e1fd68e28 100644 --- a/pkg/config/setup/system_probe_cws.go +++ b/pkg/config/setup/system_probe_cws.go @@ -57,6 +57,7 @@ func initCWSSystemProbeConfig(cfg pkgconfigmodel.Config) { cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.min_timeout", "10m") cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.max_dump_size", 1750) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_cgroups_count", 5) + cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.cgroup_managers", []string{"docker", "podman", "containerd", "cri-o"}) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_event_types", []string{"exec", "open", "dns", "imds"}) cfg.BindEnv("runtime_security_config.activity_dump.cgroup_dump_timeout") // deprecated in favor of dump_duration cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.dump_duration", "900s") diff --git a/pkg/fleet/daemon/daemon.go b/pkg/fleet/daemon/daemon.go index ba79d5456dd3e..5023393d99e5b 100644 --- a/pkg/fleet/daemon/daemon.go +++ b/pkg/fleet/daemon/daemon.go @@ -570,7 +570,7 @@ type requestState struct { ErrorCode installerErrors.InstallerErrorCode } -func newRequestContext(request remoteAPIRequest) (telemetry.Span, context.Context) { +func newRequestContext(request remoteAPIRequest) (*telemetry.Span, context.Context) { ctx := context.WithValue(context.Background(), requestStateKey, &requestState{ Package: request.Package, ID: request.ID, diff --git a/pkg/fleet/installer/oci/download.go b/pkg/fleet/installer/oci/download.go index 7fcce0234512c..e557cf8b53d7f 100644 --- a/pkg/fleet/installer/oci/download.go +++ b/pkg/fleet/installer/oci/download.go @@ -62,8 +62,8 @@ const ( ) const ( - layerMaxSize = 3 << 30 // 3GiB - extractLayerRetries = 3 + layerMaxSize = 3 << 30 // 3GiB + networkRetries = 3 ) var ( @@ -318,30 +318,32 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) return fmt.Errorf("could not get layer media type: %w", err) } if layerMediaType == mediaType { - // Retry stream reset errors - for i := 0; i < extractLayerRetries; i++ { - if i > 0 { - time.Sleep(time.Second) - } - uncompressedLayer, err := layer.Uncompressed() - if err != nil { - return fmt.Errorf("could not uncompress layer: %w", err) - } - err = tar.Extract(uncompressedLayer, dir, layerMaxSize) - uncompressedLayer.Close() - if err != nil { - if !isStreamResetError(err) && !isConnectionResetByPeerError(err) { - return fmt.Errorf("could not extract layer: %w", err) + err = withNetworkRetries( + func() error { + var err error + defer func() { + if err != nil { + deferErr := tar.Clean(dir) + if deferErr != nil { + err = deferErr + } + } + }() + uncompressedLayer, err := layer.Uncompressed() + if err != nil { + return err } - log.Warnf("network error while extracting layer, retrying") - // Clean up the directory before retrying to avoid partial extraction - err = tar.Clean(dir) + err = tar.Extract(uncompressedLayer, dir, layerMaxSize) + uncompressedLayer.Close() if err != nil { - return fmt.Errorf("could not clean directory: %w", err) + return err } - } else { - break - } + + return nil + }, + ) + if err != nil { + return fmt.Errorf("could not extract layer: %w", err) } } } @@ -349,16 +351,22 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) } // WriteOCILayout writes the image as an OCI layout to the given directory. -func (d *DownloadedPackage) WriteOCILayout(dir string) error { - layoutPath, err := layout.Write(dir, empty.Index) - if err != nil { - return fmt.Errorf("could not write layout: %w", err) - } - err = layoutPath.AppendImage(d.Image) - if err != nil { - return fmt.Errorf("could not append image to layout: %w", err) - } - return nil +func (d *DownloadedPackage) WriteOCILayout(dir string) (err error) { + var layoutPath layout.Path + return withNetworkRetries( + func() error { + layoutPath, err = layout.Write(dir, empty.Index) + if err != nil { + return fmt.Errorf("could not write layout: %w", err) + } + + err = layoutPath.AppendImage(d.Image) + if err != nil { + return fmt.Errorf("could not append image to layout: %w", err) + } + return nil + }, + ) } // PackageURL returns the package URL for the given site, package and version. @@ -371,15 +379,50 @@ func PackageURL(env *env.Env, pkg string, version string) string { } } +func withNetworkRetries(f func() error) error { + var err error + for i := 0; i < networkRetries; i++ { + err = f() + if err == nil { + return nil + } + if !isRetryableNetworkError(err) { + return err + } + log.Warnf("retrying after network error: %s", err) + time.Sleep(time.Second) + } + return err +} + +// isRetryableNetworkError returns true if the error is a network error we should retry on +func isRetryableNetworkError(err error) bool { + if err == nil { + return false + } + + if netErr, ok := err.(*net.OpError); ok { + if netErr.Temporary() { + // Temporary errors, such as "connection timed out" + return true + } + if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { + if errno, ok := syscallErr.Err.(syscall.Errno); ok { + // Connection reset errors, such as "connection reset by peer" + return errno == syscall.ECONNRESET + } + } + } + + return isStreamResetError(err) +} + // isStreamResetError returns true if the given error is a stream reset error. // Sometimes, in GCR, the tar extract fails with "stream error: stream ID x; INTERNAL_ERROR; received from peer". // This happens because the uncompressed layer reader is a http/2 response body under the hood. That body is // streamed and receives a "reset stream frame", with the code 0x2 (INTERNAL_ERROR). This is an error from the server // that we need to retry. func isStreamResetError(err error) bool { - if err == nil { - return false - } serr := http2.StreamError{} if errors.As(err, &serr) { return serr.Code == http2.ErrCodeInternal @@ -391,18 +434,6 @@ func isStreamResetError(err error) bool { return false } -// isConnectionResetByPeer returns true if the error is a connection reset by peer error -func isConnectionResetByPeerError(err error) bool { - if netErr, ok := err.(*net.OpError); ok { - if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { - if errno, ok := syscallErr.Err.(syscall.Errno); ok { - return errno == syscall.ECONNRESET - } - } - } - return false -} - type usernamePasswordKeychain struct { username string password string diff --git a/pkg/fleet/installer/setup/common/setup.go b/pkg/fleet/installer/setup/common/setup.go index 0a7b93447c22f..b6e512d7dc5d6 100644 --- a/pkg/fleet/installer/setup/common/setup.go +++ b/pkg/fleet/installer/setup/common/setup.go @@ -41,7 +41,7 @@ type Setup struct { Out *Output Env *env.Env Ctx context.Context - Span telemetry.Span + Span *telemetry.Span Packages Packages Config Config } @@ -130,7 +130,7 @@ func (s *Setup) installPackage(name string, url string) (err error) { span, ctx := telemetry.StartSpanFromContext(s.Ctx, "install") defer func() { span.Finish(err) }() span.SetTag("url", url) - span.SetTag("_top_level", 1) + span.SetTopLevel() s.Out.WriteString(fmt.Sprintf("Installing %s...\n", name)) err = s.installer.Install(ctx, url, nil) diff --git a/pkg/fleet/internal/exec/installer_exec.go b/pkg/fleet/internal/exec/installer_exec.go index 834f54b3d8a53..92f5880723fa9 100644 --- a/pkg/fleet/internal/exec/installer_exec.go +++ b/pkg/fleet/internal/exec/installer_exec.go @@ -40,7 +40,7 @@ func NewInstallerExec(env *env.Env, installerBinPath string) *InstallerExec { type installerCmd struct { *exec.Cmd - span telemetry.Span + span *telemetry.Span ctx context.Context } diff --git a/pkg/fleet/telemetry/http_wrapper.go b/pkg/fleet/telemetry/http_wrapper.go new file mode 100644 index 0000000000000..2f94bda79fe1f --- /dev/null +++ b/pkg/fleet/telemetry/http_wrapper.go @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "net/http" + "strconv" + "strings" +) + +// WrapRoundTripper wraps the round tripper with the telemetry round tripper. +func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { + if rt == nil { + rt = http.DefaultTransport + } + if wrapped, ok := rt.(*roundTripper); ok { + rt = wrapped.base + } + return &roundTripper{ + base: rt, + } +} + +type roundTripper struct { + base http.RoundTripper +} + +func (rt *roundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + span, _ := StartSpanFromContext(req.Context(), "http.request") + defer func() { span.Finish(err) }() + + url := *req.URL + url.User = nil + + span.span.Type = "http" + span.SetResourceName(req.Method + " " + urlFromRequest(req)) + span.span.Meta["http.method"] = req.Method + span.span.Meta["http.url"] = req.URL.String() + span.span.Meta["span.kind"] = "client" + span.span.Meta["network.destination.name"] = url.Hostname() + res, err = rt.base.RoundTrip(req) + if err != nil { + span.SetTag("http.errors", err.Error()) + return res, err + } + span.SetTag("http.status_code", strconv.Itoa(res.StatusCode)) + if res.StatusCode >= 400 { + span.SetTag("http.errors", res.Status) + } + return res, err +} + +// urlFromRequest returns the URL from the HTTP request. The URL query string is included in the return object iff queryString is true +// See https://docs.datadoghq.com/tracing/configure_data_security#redacting-the-query-in-the-url for more information. +func urlFromRequest(r *http.Request) string { + // Quoting net/http comments about net.Request.URL on server requests: + // "For most requests, fields other than Path and RawQuery will be + // empty. (See RFC 7230, Section 5.3)" + // This is why we don't rely on url.URL.String(), url.URL.Host, url.URL.Scheme, etc... + var url string + path := r.URL.EscapedPath() + scheme := r.URL.Scheme + if r.TLS != nil { + scheme = "https" + } + if r.Host != "" { + url = strings.Join([]string{scheme, "://", r.Host, path}, "") + } else { + url = path + } + // Collect the query string if we are allowed to report it and obfuscate it if possible/allowed + if r.URL.RawQuery != "" { + query := r.URL.RawQuery + url = strings.Join([]string{url, query}, "?") + } + if frag := r.URL.EscapedFragment(); frag != "" { + url = strings.Join([]string{url, frag}, "#") + } + return url +} diff --git a/pkg/fleet/telemetry/span.go b/pkg/fleet/telemetry/span.go index ac8f79516c736..ac5f00b9ade0a 100644 --- a/pkg/fleet/telemetry/span.go +++ b/pkg/fleet/telemetry/span.go @@ -7,20 +7,144 @@ package telemetry import ( - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "context" + "fmt" + "math/rand/v2" + "runtime/debug" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" ) -// Span is an alias for ddtrace.Span until we phase ddtrace out. -type Span struct{ ddtrace.Span } +const spanKey = spanContextKey("span_context") + +type spanContextKey string + +// Span represents a span. +type Span struct { + mu sync.Mutex + span internaltelemetry.Span + finished atomic.Bool +} + +func newSpan(name string, parentID, traceID uint64) *Span { + if traceID == 0 { + traceID = rand.Uint64() + if !headSamplingKeep(name, traceID) { + traceID = dropTraceID + } + } + s := &Span{ + span: internaltelemetry.Span{ + TraceID: traceID, + ParentID: parentID, + SpanID: rand.Uint64(), + Name: name, + Resource: name, + Start: time.Now().UnixNano(), + Meta: make(map[string]string), + Metrics: make(map[string]float64), + }, + } + if parentID == 0 { + s.SetTopLevel() + } + + globalTracer.registerSpan(s) + return s +} // Finish finishes the span with an error. func (s *Span) Finish(err error) { - s.Span.Finish(tracer.WithError(err)) + s.finished.Store(true) + s.mu.Lock() + defer s.mu.Unlock() + s.span.Duration = time.Now().UnixNano() - s.span.Start + if err != nil { + s.span.Error = 1 + s.span.Meta = map[string]string{ + "error.message": err.Error(), + "error.stack": string(debug.Stack()), + } + } + globalTracer.finishSpan(s) } // SetResourceName sets the resource name of the span. func (s *Span) SetResourceName(name string) { - s.Span.SetTag(ext.ResourceName, name) + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.span.Resource = name +} + +// SetTopLevel sets the span as a top level span. +func (s *Span) SetTopLevel() { + s.SetTag("_top_level", 1) +} + +// SetTag sets a tag on the span. +func (s *Span) SetTag(key string, value interface{}) { + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if value == nil { + s.span.Meta[key] = "nil" + } + switch v := value.(type) { + case string: + s.span.Meta[key] = v + case bool: + s.span.Meta[key] = strconv.FormatBool(v) + case int: + s.span.Metrics[key] = float64(v) + case int8: + s.span.Metrics[key] = float64(v) + case int16: + s.span.Metrics[key] = float64(v) + case int32: + s.span.Metrics[key] = float64(v) + case int64: + s.span.Metrics[key] = float64(v) + case uint: + s.span.Metrics[key] = float64(v) + case uint8: + s.span.Metrics[key] = float64(v) + case uint16: + s.span.Metrics[key] = float64(v) + case uint32: + s.span.Metrics[key] = float64(v) + case uint64: + s.span.Metrics[key] = float64(v) + case float32: + s.span.Metrics[key] = float64(v) + case float64: + s.span.Metrics[key] = v + default: + s.span.Meta[key] = fmt.Sprintf("not_supported_type %T", v) + } +} + +type spanIDs struct { + traceID uint64 + spanID uint64 +} + +func getSpanIDsFromContext(ctx context.Context) (spanIDs, bool) { + sIDs, ok := ctx.Value(spanKey).(spanIDs) + if !ok { + return spanIDs{}, false + } + return sIDs, true +} + +func setSpanIDsInContext(ctx context.Context, span *Span) context.Context { + return context.WithValue(ctx, spanKey, spanIDs{traceID: span.span.TraceID, spanID: span.span.SpanID}) } diff --git a/pkg/fleet/telemetry/telemetry.go b/pkg/fleet/telemetry/telemetry.go index 5bbfd8ca773c4..66174ad18fc32 100644 --- a/pkg/fleet/telemetry/telemetry.go +++ b/pkg/fleet/telemetry/telemetry.go @@ -8,280 +8,178 @@ package telemetry import ( "context" - "errors" "fmt" - "io" - "math/rand/v2" - "net" "net/http" "os" "strconv" "strings" - "sync" - - httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" - - "github.com/gorilla/mux" + "time" "github.com/DataDog/datadog-agent/pkg/internaltelemetry" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - traceconfig "github.com/DataDog/datadog-agent/pkg/trace/config" - "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" ) const ( - // EnvTraceID is the environment variable key for the trace ID - EnvTraceID = "DATADOG_TRACE_ID" - // EnvParentID is the environment variable key for the parent ID - EnvParentID = "DATADOG_PARENT_ID" -) - -const ( + envTraceID = "DATADOG_TRACE_ID" + envParentID = "DATADOG_PARENT_ID" telemetrySubdomain = "instrumentation-telemetry-intake" - telemetryEndpoint = "/v0.4/traces" ) // Telemetry handles the telemetry for fleet components. type Telemetry struct { telemetryClient internaltelemetry.Client + done chan struct{} + flushed chan struct{} - site string + env string service string - - listener *telemetryListener - server *http.Server - client *http.Client - - samplingRules []tracer.SamplingRule } -// Option is a functional option for telemetry. -type Option func(*Telemetry) - // NewTelemetry creates a new telemetry instance -func NewTelemetry(client *http.Client, apiKey string, site string, service string, opts ...Option) *Telemetry { - endpoint := &traceconfig.Endpoint{ +func NewTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + t := newTelemetry(client, apiKey, site, service) + t.Start() + return t +} + +func newTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + endpoint := &internaltelemetry.Endpoint{ Host: fmt.Sprintf("https://%s.%s", telemetrySubdomain, strings.TrimSpace(site)), APIKey: apiKey, } - listener := newTelemetryListener() - t := &Telemetry{ - telemetryClient: internaltelemetry.NewClient(client, []*traceconfig.Endpoint{endpoint}, service, site == "datad0g.com"), - site: site, - service: service, - listener: listener, - server: &http.Server{}, - client: &http.Client{ - Transport: &http.Transport{ - Dial: listener.Dial, - }, - }, + env := "prod" + if site == "datad0g.com" { + env = "staging" } - for _, opt := range opts { - opt(t) + + return &Telemetry{ + telemetryClient: internaltelemetry.NewClient(client, []*internaltelemetry.Endpoint{endpoint}, service, site == "datad0g.com"), + done: make(chan struct{}), + flushed: make(chan struct{}), + env: env, + service: service, } - t.server.Handler = t.handler() - return t } // Start starts the telemetry -func (t *Telemetry) Start(_ context.Context) error { +func (t *Telemetry) Start() { + ticker := time.Tick(1 * time.Minute) go func() { - err := t.server.Serve(t.listener) - if err != nil { - log.Infof("telemetry server stopped: %v", err) + for { + select { + case <-ticker: + t.sendCompletedSpans() + case <-t.done: + t.sendCompletedSpans() + close(t.flushed) + return + } } }() - env := "prod" - if t.site == "datad0g.com" { - env = "staging" - } - - tracer.Start( - tracer.WithService(t.service), - tracer.WithServiceVersion(version.AgentVersion), - tracer.WithEnv(env), - tracer.WithGlobalTag("site", t.site), - tracer.WithHTTPClient(t.client), - tracer.WithLogStartup(false), - - // We don't need the value, we just need to enforce that it's not - // the default. If it is, then the tracer will try to use the socket - // if it exists -- and it always exists for newer agents. - // If the agent address is the socket, the tracer overrides WithHTTPClient to use it. - tracer.WithAgentAddr("192.0.2.42:12345"), // 192.0.2.0/24 is reserved - tracer.WithSamplingRules(t.samplingRules), - ) - return nil } // Stop stops the telemetry -func (t *Telemetry) Stop(ctx context.Context) error { - tracer.Flush() - tracer.Stop() - t.listener.Close() - err := t.server.Shutdown(ctx) - if err != nil { - log.Errorf("error shutting down telemetry server: %v", err) - } - return nil -} - -func (t *Telemetry) handler() http.Handler { - r := mux.NewRouter().Headers("Content-Type", "application/msgpack").Subrouter() - r.HandleFunc(telemetryEndpoint, func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - log.Errorf("error reading request body: %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - var traces pb.Traces - _, err = traces.UnmarshalMsg(body) - if err != nil { - log.Errorf("error unmarshalling traces: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - t.telemetryClient.SendTraces(traces) - w.WriteHeader(http.StatusOK) - }) - return r +func (t *Telemetry) Stop() { + close(t.done) + <-t.flushed } -type telemetryListener struct { - conns chan net.Conn - - close chan struct{} - closeOnce sync.Once -} - -func newTelemetryListener() *telemetryListener { - return &telemetryListener{ - conns: make(chan net.Conn), - close: make(chan struct{}), +func (t *Telemetry) extractCompletedSpans() internaltelemetry.Traces { + spans := globalTracer.flushCompletedSpans() + if len(spans) == 0 { + return internaltelemetry.Traces{} } -} - -func (l *telemetryListener) Close() error { - l.closeOnce.Do(func() { - close(l.close) - }) - return nil -} - -func (l *telemetryListener) Accept() (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - case conn := <-l.conns: - return conn, nil + traces := make(map[uint64][]*internaltelemetry.Span) + for _, span := range spans { + span.span.Service = t.service + span.span.Meta["env"] = t.env + span.span.Meta["version"] = version.AgentVersion + span.span.Metrics["_sampling_priority_v1"] = 2 + traces[span.span.TraceID] = append(traces[span.span.TraceID], &span.span) + } + tracesArray := make([]internaltelemetry.Trace, 0, len(traces)) + for _, trace := range traces { + tracesArray = append(tracesArray, internaltelemetry.Trace(trace)) } + return internaltelemetry.Traces(tracesArray) } -func (l *telemetryListener) Addr() net.Addr { - return addr(0) +func (t *Telemetry) sendCompletedSpans() { + tracesArray := t.extractCompletedSpans() + if len(tracesArray) == 0 { + return + } + t.telemetryClient.SendTraces(tracesArray) } -func (l *telemetryListener) Dial(_, _ string) (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - default: +// SpanFromContext returns the span from the context if available. +func SpanFromContext(ctx context.Context) (*Span, bool) { + spanIDs, ok := getSpanIDsFromContext(ctx) + if !ok { + return nil, false } - server, client := net.Pipe() - l.conns <- server - return client, nil + return globalTracer.getSpan(spanIDs.spanID) } -type addr int - -func (addr) Network() string { - return "memory" +// StartSpanFromEnv starts a span using the environment variables to find the parent span. +func StartSpanFromEnv(ctx context.Context, operationName string) (*Span, context.Context) { + traceID, parentID := extractIDsFromEnv() + return StartSpanFromIDs(ctx, operationName, traceID, parentID) } -func (addr) String() string { - return "local" +func extractIDsFromEnv() (string, string) { + parentID, ok := os.LookupEnv(envParentID) + if !ok { + return "0", "0" + } + traceID, ok := os.LookupEnv(envTraceID) + if !ok { + return "0", "0" + } + return traceID, parentID } -// StartSpanFromIDs starts a span using the trace and parent -// IDs provided. -func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - ctxCarrier := tracer.TextMapCarrier{ - tracer.DefaultTraceIDHeader: traceID, - tracer.DefaultParentIDHeader: parentID, - tracer.DefaultPriorityHeader: "2", +func converIDsToUint64(traceID, parentID string) (uint64, uint64) { + traceIDInt, err := strconv.ParseUint(traceID, 10, 64) + if err != nil { + return 0, 0 } - spanCtx, err := tracer.Extract(ctxCarrier) + parentIDInt, err := strconv.ParseUint(parentID, 10, 64) if err != nil { - log.Debugf("failed to extract span context from install script params: %v", err) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return 0, 0 } - spanOptions = append([]ddtrace.StartSpanOption{tracer.ChildOf(spanCtx)}, spanOptions...) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return traceIDInt, parentIDInt } -// SpanFromContext returns the span from the context if available. -func SpanFromContext(ctx context.Context) (Span, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return Span{}, false - } - return Span{span}, true +// StartSpanFromIDs starts a span using the trace and parent +// IDs provided. +func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string) (*Span, context.Context) { + traceIDInt, parentIDInt := converIDsToUint64(traceID, parentID) + span, ctx := startSpanFromIDs(ctx, operationName, traceIDInt, parentIDInt) + span.SetTopLevel() + return span, ctx } -// StartSpanFromContext starts a span using the context to find the parent span. -func StartSpanFromContext(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - span, ctx := tracer.StartSpanFromContext(ctx, operationName, spanOptions...) - return Span{span}, ctx +func startSpanFromIDs(ctx context.Context, operationName string, traceID, parentID uint64) (*Span, context.Context) { + s := newSpan(operationName, parentID, traceID) + ctx = setSpanIDsInContext(ctx, s) + return s, ctx } -// StartSpanFromEnv starts a span using the environment variables to find the parent span. -func StartSpanFromEnv(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - traceID, ok := os.LookupEnv(EnvTraceID) - if !ok { - traceID = strconv.FormatUint(rand.Uint64(), 10) - } - parentID, ok := os.LookupEnv(EnvParentID) - if !ok { - parentID = "0" - } - return StartSpanFromIDs(ctx, operationName, traceID, parentID, spanOptions...) +// StartSpanFromContext starts a span using the context to find the parent span. +func StartSpanFromContext(ctx context.Context, operationName string) (*Span, context.Context) { + spanIDs, _ := getSpanIDsFromContext(ctx) + return startSpanFromIDs(ctx, operationName, spanIDs.traceID, spanIDs.spanID) } // EnvFromContext returns the environment variables for the context. func EnvFromContext(ctx context.Context) []string { - spanCtx, ok := SpanContextFromContext(ctx) + sIDs, ok := getSpanIDsFromContext(ctx) if !ok { return []string{} } return []string{ - fmt.Sprintf("%s=%d", EnvTraceID, spanCtx.TraceID()), - fmt.Sprintf("%s=%d", EnvParentID, spanCtx.SpanID()), + fmt.Sprintf("%s=%s", envTraceID, strconv.FormatUint(sIDs.traceID, 10)), + fmt.Sprintf("%s=%s", envParentID, strconv.FormatUint(sIDs.spanID, 10)), } } - -// SpanContextFromContext extracts the span context from the context if available. -func SpanContextFromContext(ctx context.Context) (ddtrace.SpanContext, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return nil, false - } - return span.Context(), true -} - -// WithSamplingRules sets the sampling rules for the telemetry. -func WithSamplingRules(rules ...tracer.SamplingRule) Option { - return func(t *Telemetry) { - t.samplingRules = rules - } -} - -// WrapRoundTripper wraps the round tripper with the telemetry round tripper. -func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { - return httptrace.WrapRoundTripper(rt) -} diff --git a/pkg/fleet/telemetry/telemetry_test.go b/pkg/fleet/telemetry/telemetry_test.go new file mode 100644 index 0000000000000..d0866ee4d4a13 --- /dev/null +++ b/pkg/fleet/telemetry/telemetry_test.go @@ -0,0 +1,222 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "context" + "errors" + "net/http" + "os" + "testing" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFreshSpan(t *testing.T) { + ctx := context.Background() + _, ok := SpanFromContext(ctx) + require.False(t, ok) + + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + s.SetResourceName("new") + + span, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, s, span) + + assert.Equal(t, "test", s.span.Name) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", span.span.Resource) +} + +func TestInheritence(t *testing.T) { + ctx := context.Background() + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + + child, _ := StartSpanFromContext(ctx, "child") + require.NotNil(t, child) + + assert.Equal(t, s.span.SpanID, child.span.ParentID) + assert.Equal(t, s.span.TraceID, child.span.TraceID) +} + +func TestStartSpanFromIDs(t *testing.T) { + ctx := context.Background() + traceID := "100" + parentID := "200" + + span, ctx := StartSpanFromIDs(ctx, "ids-operation", traceID, parentID) + require.NotNil(t, span, "Expected a span") + require.Equal(t, uint64(100), span.span.TraceID) + require.Equal(t, uint64(200), span.span.ParentID) + + val, ok := span.span.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + + spanFromCtx, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, span, spanFromCtx) +} + +func strPtr(s string) *string { + return &s +} + +func TestSpanFromEnv(t *testing.T) { + randTraceID := uint64(9) + tt := []struct { + name string + envTraceID *string + envParentID *string + expectedTraceID uint64 + expectedParentID uint64 + }{ + { + name: "no parent env", + envTraceID: strPtr("100"), + envParentID: nil, + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "no trace env", + envTraceID: nil, + envParentID: strPtr("100"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "traceID malformed", + envTraceID: strPtr("not-a-number"), + envParentID: strPtr("200"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "parentID malformed", + envTraceID: strPtr("100"), + envParentID: strPtr("not-a-number"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "inheritance", + envTraceID: strPtr("100"), + envParentID: strPtr("200"), + expectedTraceID: 100, + expectedParentID: 200, + }, + } + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + if tc.envTraceID != nil { + os.Setenv(envTraceID, *tc.envTraceID) + defer os.Unsetenv(envTraceID) + } + if tc.envParentID != nil { + os.Setenv(envParentID, *tc.envParentID) + defer os.Unsetenv(envParentID) + } + + span, ctx := StartSpanFromEnv(context.Background(), "env-operation") + require.NotNil(t, span, "Expected a span") + s, ok := SpanFromContext(ctx) + assert.True(t, ok) + assert.Equal(t, span, s) + + assert.Equal(t, tc.expectedParentID, span.span.ParentID) + if tc.expectedTraceID != randTraceID { + assert.Equal(t, tc.expectedTraceID, span.span.TraceID) + } else { + assert.NotEqual(t, 0, span.span.TraceID) + } + + }) + } +} + +func TestLimit(t *testing.T) { + totalSpans := maxSpansInFlight + 2 + ctx := context.Background() + for i := 0; i < totalSpans; i++ { + _, ctx = StartSpanFromContext(ctx, "test") + } + assert.Len(t, globalTracer.spans, maxSpansInFlight) +} + +func TestEnvFromContext(t *testing.T) { + s, ctx := StartSpanFromContext(context.Background(), "test") + s.span.TraceID = 456 + s.span.SpanID = 123 + ctx = setSpanIDsInContext(ctx, s) + env := EnvFromContext(ctx) + assert.ElementsMatch(t, []string{"DATADOG_TRACE_ID=456", "DATADOG_PARENT_ID=123"}, env) + + env = EnvFromContext(context.Background()) + assert.ElementsMatch(t, []string{}, env) +} + +func TestSpanFinished(t *testing.T) { + s, _ := StartSpanFromContext(context.Background(), "test") + s.Finish(nil) + s.SetResourceName("new") + s.SetTag("key", "value") + + assert.Equal(t, "test", s.span.Resource) + _, ok := s.span.Meta["key"] + assert.False(t, ok) +} + +func TestRemapOnFlush(t *testing.T) { + const testService = "test-service" + const numTraces = 10 + telem := newTelemetry(&http.Client{}, "api", "datad0g.com", testService) + globalTracer = &tracer{spans: make(map[uint64]*Span)} + + // traces with 2 spans + for i := 0; i < numTraces; i++ { + parentSpan, ctx := StartSpanFromContext(context.Background(), "parent") + childSpan, _ := StartSpanFromContext(ctx, "child") + childSpan.Finish(errors.New("test_error")) + parentSpan.Finish(nil) + } + resTraces := telem.extractCompletedSpans() + require.Len(t, resTraces, numTraces) + + for _, trace := range resTraces { + assert.Len(t, trace, 2) + for _, span := range trace { + assert.Equal(t, testService, span.Service) + assert.Equal(t, "staging", span.Meta["env"]) + assert.Equal(t, 2.0, span.Metrics["_sampling_priority_v1"]) + } + var parent, child *internaltelemetry.Span + if trace[0].Name == "parent" { + parent = trace[0] + child = trace[1] + } else { + parent = trace[1] + child = trace[0] + } + assert.Equal(t, parent.SpanID, child.ParentID) + val, ok := parent.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + _, ok = child.Metrics["_top_level"] + require.False(t, ok) + + require.Equal(t, int32(1), child.Error) + require.Equal(t, "test_error", child.Meta["error.message"]) + require.Contains(t, child.Meta["error.stack"], "telemetry_test.go") + } +} diff --git a/pkg/fleet/telemetry/tracer.go b/pkg/fleet/telemetry/tracer.go new file mode 100644 index 0000000000000..0923b3563e457 --- /dev/null +++ b/pkg/fleet/telemetry/tracer.go @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "math" + "strings" + "sync" +) + +const ( + dropTraceID = 1 + maxSpansInFlight = 1000 +) + +var ( + globalTracer *tracer + samplingRates = map[string]float64{ + "cdn": 0.1, + "garbage_collect": 0.05, + "HTTPClient": 0.05, + } +) + +func init() { + globalTracer = &tracer{ + spans: make(map[uint64]*Span), + } +} + +type tracer struct { + mu sync.Mutex + spans map[uint64]*Span + completedSpans []*Span +} + +func (t *tracer) registerSpan(span *Span) { + if span.span.TraceID == dropTraceID { + return + } + t.mu.Lock() + defer t.mu.Unlock() + // naive maxSpansInFlight check as this is just telemetry + // next iteration if needed would be to flush long running spans to troubleshoot + if len(t.spans) >= maxSpansInFlight { + return + } + t.spans[span.span.SpanID] = span +} + +func (t *tracer) getSpan(spanID uint64) (*Span, bool) { + t.mu.Lock() + defer t.mu.Unlock() + span, ok := t.spans[spanID] + return span, ok +} + +func (t *tracer) finishSpan(span *Span) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.spans, span.span.SpanID) + t.completedSpans = append(t.completedSpans, span) +} + +func (t *tracer) flushCompletedSpans() []*Span { + t.mu.Lock() + defer t.mu.Unlock() + newSpanArray := make([]*Span, 0) + completedSpans := t.completedSpans + t.completedSpans = newSpanArray + return completedSpans +} + +func headSamplingKeep(spanName string, traceID uint64) bool { + for k, r := range samplingRates { + if strings.Contains(spanName, k) { + return sampledByRate(traceID, r) + } + } + return true +} + +func sampledByRate(n uint64, rate float64) bool { + if rate < 1 { + return n*uint64(1111111111111111111) < uint64(rate*math.MaxUint64) + } + return true +} diff --git a/pkg/internaltelemetry/client.go b/pkg/internaltelemetry/client.go index 52b1b2010edf0..ee7754fad190b 100644 --- a/pkg/internaltelemetry/client.go +++ b/pkg/internaltelemetry/client.go @@ -21,8 +21,6 @@ import ( "go.uber.org/atomic" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - "github.com/DataDog/datadog-agent/pkg/trace/config" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" "github.com/shirou/gopsutil/v4/host" @@ -36,13 +34,19 @@ const ( // Client defines the interface for a telemetry client type Client interface { SendLog(level string, message string) - SendTraces(traces pb.Traces) + SendTraces(traces Traces) +} + +// Endpoint defines the endpoint object +type Endpoint struct { + APIKey string `json:"-"` + Host string } type client struct { m sync.Mutex client httpClient - endpoints []*config.Endpoint + endpoints []*Endpoint sendPayloadTimeout time.Duration // we can pre-calculate the host payload structure at init time @@ -94,7 +98,7 @@ type Application struct { // TracePayload defines the trace payload object type TracePayload struct { - Traces []pb.Trace `json:"traces"` + Traces []Trace `json:"traces"` } // LogPayload defines the log payload object @@ -115,7 +119,7 @@ type httpClient interface { } // NewClient creates a new telemetry client -func NewClient(httpClient httpClient, endpoints []*config.Endpoint, service string, debug bool) Client { +func NewClient(httpClient httpClient, endpoints []*Endpoint, service string, debug bool) Client { info, err := host.Info() if err != nil { log.Errorf("failed to retrieve host info: %v", err) @@ -158,7 +162,7 @@ func (c *client) SendLog(level, message string) { c.sendPayload(RequestTypeLogs, payload) } -func (c *client) SendTraces(traces pb.Traces) { +func (c *client) SendTraces(traces Traces) { c.m.Lock() defer c.m.Unlock() payload := TracePayload{ @@ -170,8 +174,8 @@ func (c *client) SendTraces(traces pb.Traces) { // sampleTraces is a simple uniform sampling function that samples traces based // on the sampling rate, given that there is no trace agent to sample the traces // We try to keep the tracer behaviour: the first rule that matches apply its rate to the whole trace -func (c *client) sampleTraces(traces pb.Traces) pb.Traces { - tracesWithSampling := pb.Traces{} +func (c *client) sampleTraces(traces Traces) Traces { + tracesWithSampling := Traces{} for _, trace := range traces { samplingRate := 1.0 for _, span := range trace { @@ -206,7 +210,7 @@ func (c *client) sendPayload(requestType RequestType, payload interface{}) { group := sync.WaitGroup{} for _, endpoint := range c.endpoints { group.Add(1) - go func(endpoint *config.Endpoint) { + go func(endpoint *Endpoint) { defer group.Done() url := fmt.Sprintf("%s%s", endpoint.Host, telemetryEndpoint) req, err := http.NewRequest("POST", url, bytes.NewReader(serializedPayload)) diff --git a/pkg/internaltelemetry/traces.go b/pkg/internaltelemetry/traces.go new file mode 100644 index 0000000000000..54c8545a604b7 --- /dev/null +++ b/pkg/internaltelemetry/traces.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +// Package internaltelemetry full description in README.md +package internaltelemetry + +// Traces is a collection of traces +type Traces []Trace + +// Trace is a collection of spans with the same trace ID +type Trace []*Span + +// Span used for installation telemetry +type Span struct { + // Service is the name of the service that handled this span. + Service string `json:"service"` + // Name is the name of the operation this span represents. + Name string `json:"name"` + // Resource is the name of the resource this span represents. + Resource string `json:"resource"` + // TraceID is the ID of the trace to which this span belongs. + TraceID uint64 `json:"trace_id"` + // SpanID is the ID of this span. + SpanID uint64 `json:"span_id"` + // ParentID is the ID of the parent span. + ParentID uint64 `json:"parent_id"` + // Start is the start time of this span in nanoseconds since the Unix epoch. + Start int64 `json:"start"` + // Duration is the duration of this span in nanoseconds. + Duration int64 `json:"duration"` + // Error is the error status of this span. + Error int32 `json:"error"` + // Meta is a mapping from tag name to tag value for string-valued tags. + Meta map[string]string `json:"meta,omitempty"` + // Metrics is a mapping from metric name to metric value for numeric metrics. + Metrics map[string]float64 `json:"metrics,omitempty"` + // Type is the type of the span. + Type string `json:"type"` +} diff --git a/pkg/proto/pbgo/core/model.pb.go b/pkg/proto/pbgo/core/model.pb.go index 016a2cf53e260..9262b4dc43a5a 100644 --- a/pkg/proto/pbgo/core/model.pb.go +++ b/pkg/proto/pbgo/core/model.pb.go @@ -1155,8 +1155,8 @@ type GenerateContainerIDFromOriginInfoRequest_ExternalData struct { unknownFields protoimpl.UnknownFields Init *bool `protobuf:"varint,1,opt,name=init,proto3,oneof" json:"init,omitempty"` // Init is true if the container is an init container. - ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name as seen by the Admission Controller. - PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID as seen by the Admission Controller. + ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name in the Kubernetes Pod spec. + PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID in the Kubernetes Pod spec. } func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) Reset() { diff --git a/pkg/security/config/config.go b/pkg/security/config/config.go index ab6b0b6b591e1..9d165a4ae1900 100644 --- a/pkg/security/config/config.go +++ b/pkg/security/config/config.go @@ -101,6 +101,9 @@ type RuntimeSecurityConfig struct { // ActivityDumpTracedCgroupsCount defines the maximum count of cgroups that should be monitored concurrently. Leave this parameter to 0 to prevent the generation // of activity dumps based on cgroups. ActivityDumpTracedCgroupsCount int + // ActivityDumpCgroupsManagers defines the cgroup managers we generate dumps for. + ActivityDumpCgroupsManagers []string + // ActivityDumpTracedEventTypes defines the list of events that should be captured in an activity dump. Leave this // parameter empty to monitor all event types. If not already present, the `exec` event will automatically be added // to this list. @@ -368,6 +371,7 @@ func NewRuntimeSecurityConfig() (*RuntimeSecurityConfig, error) { ActivityDumpLoadControlPeriod: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.load_controller_period"), ActivityDumpLoadControlMinDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.min_timeout"), ActivityDumpTracedCgroupsCount: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.traced_cgroups_count"), + ActivityDumpCgroupsManagers: pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.cgroup_managers"), ActivityDumpTracedEventTypes: parseEventTypeStringSlice(pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.traced_event_types")), ActivityDumpCgroupDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.dump_duration"), ActivityDumpRateLimiter: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.rate_limiter"), diff --git a/pkg/security/ebpf/c/include/constants/custom.h b/pkg/security/ebpf/c/include/constants/custom.h index 88be17fa3c80b..1e2311bf8fc7e 100644 --- a/pkg/security/ebpf/c/include/constants/custom.h +++ b/pkg/security/ebpf/c/include/constants/custom.h @@ -196,4 +196,8 @@ static __attribute__((always_inline)) u64 get_imds_ip() { #define CGROUP_MANAGER_CRI 4 #define CGROUP_MANAGER_SYSTEMD 5 +#define CGROUP_MANAGER_MASK 0b111 +#define CGROUP_SYSTEMD_SERVICE (0 << 8) +#define CGROUP_SYSTEMD_SCOPE (1 << 8) + #endif diff --git a/pkg/security/ebpf/c/include/helpers/activity_dump.h b/pkg/security/ebpf/c/include/helpers/activity_dump.h index c0f8d246006bf..70e776cca8161 100644 --- a/pkg/security/ebpf/c/include/helpers/activity_dump.h +++ b/pkg/security/ebpf/c/include/helpers/activity_dump.h @@ -53,10 +53,16 @@ __attribute__((always_inline)) struct cgroup_tracing_event_t *get_cgroup_tracing return evt; } +__attribute__((always_inline)) u32 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { + u32 cgroup_manager = cgroup->cgroup_flags & CGROUP_MANAGER_MASK; + u32 supported = (cgroup->cgroup_flags != 0) && (bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_manager) != NULL); + return supported; +} + __attribute__((always_inline)) bool reserve_traced_cgroup_spot(struct cgroup_context_t *cgroup, u64 now, u64 cookie, struct activity_dump_config *config) { // insert dump config defaults - u32 defaults_key = 0; - struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &defaults_key); + u32 cgroup_flags = cgroup->cgroup_flags; + struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_flags); if (defaults == NULL) { // should never happen, ignore return false; @@ -102,11 +108,15 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return 0; } - if ((container->cgroup_context.cgroup_flags & 0b111) == CGROUP_MANAGER_SYSTEMD) { + if (!is_cgroup_activity_dumps_supported(&container->cgroup_context)) { return 0; } - copy_container_id(container->container_id, evt->container.container_id); + if ((container->cgroup_context.cgroup_flags&CGROUP_MANAGER_MASK) != CGROUP_MANAGER_SYSTEMD) { + copy_container_id(container->container_id, evt->container.container_id); + } else { + evt->container.container_id[0] = '\0'; + } evt->container.cgroup_context = container->cgroup_context; evt->cookie = cookie; evt->config = config; @@ -115,10 +125,6 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return cookie; } -__attribute__((always_inline)) u64 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { - return (cgroup->cgroup_flags != 0) && ((cgroup->cgroup_flags&0b111) != CGROUP_MANAGER_SYSTEMD); -} - __attribute__((always_inline)) u64 should_trace_new_process_cgroup(void *ctx, u64 now, u32 pid, struct container_context_t *container) { // should we start tracing this cgroup ? struct cgroup_context_t cgroup_context; diff --git a/pkg/security/ebpf/c/include/hooks/cgroup.h b/pkg/security/ebpf/c/include/hooks/cgroup.h index a231142b8f90a..b7ce66a870b92 100644 --- a/pkg/security/ebpf/c/include/hooks/cgroup.h +++ b/pkg/security/ebpf/c/include/hooks/cgroup.h @@ -189,13 +189,13 @@ static __attribute__((always_inline)) int trace__cgroup_write(ctx_t *ctx) { #endif int length = bpf_probe_read_str(prefix, sizeof(cgroup_prefix_t), container_id) & 0xff; - if (cgroup_flags == 0 && ( - (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') - || - (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') - )) { - cgroup_flags = CGROUP_MANAGER_SYSTEMD; - } else if (cgroup_flags != 0) { + if (cgroup_flags == 0) { + if (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SERVICE; + } else if (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SCOPE; + } + } else { bpf_probe_read(&new_entry.container.container_id, sizeof(new_entry.container.container_id), container_id); } diff --git a/pkg/security/ebpf/c/include/maps.h b/pkg/security/ebpf/c/include/maps.h index d6ab3ceb74dfb..3b821b19fe5cf 100644 --- a/pkg/security/ebpf/c/include/maps.h +++ b/pkg/security/ebpf/c/include/maps.h @@ -28,7 +28,7 @@ BPF_ARRAY_MAP(syscall_ctx_gen_id, u32, 1) BPF_ARRAY_MAP(syscall_ctx, char[MAX_SYSCALL_CTX_SIZE], MAX_SYSCALL_CTX_ENTRIES) BPF_HASH_MAP(activity_dumps_config, u64, struct activity_dump_config, 1) // max entries will be overridden at runtime -BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 1) +BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 5) BPF_HASH_MAP(traced_cgroups, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(cgroup_wait_list, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(traced_pids, u32, u64, 8192) // max entries will be overridden at runtime diff --git a/pkg/security/probe/probe_ebpf.go b/pkg/security/probe/probe_ebpf.go index f06cbe8fa45ae..5fd26b4f63f8a 100644 --- a/pkg/security/probe/probe_ebpf.go +++ b/pkg/security/probe/probe_ebpf.go @@ -818,15 +818,11 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)); err != nil { + cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)) + if err != nil { seclog.Debugf("Failed to resolve cgroup: %s", err) } else { event.CgroupTracing.CGroupContext = *cgroupContext - if cgroupContext.CGroupFlags.IsContainer() { - containerID, _ := containerutils.FindContainerID(cgroupContext.CGroupID) - event.CgroupTracing.ContainerContext.ContainerID = containerID - } - p.profileManagers.activityDumpManager.HandleCGroupTracingEvent(&event.CgroupTracing) } diff --git a/pkg/security/resolvers/cgroup/resolver.go b/pkg/security/resolvers/cgroup/resolver.go index 2137cccf14506..dcb1de1e6d378 100644 --- a/pkg/security/resolvers/cgroup/resolver.go +++ b/pkg/security/resolvers/cgroup/resolver.go @@ -48,6 +48,7 @@ type ResolverInterface interface { type Resolver struct { *utils.Notifier[Event, *cgroupModel.CacheEntry] sync.Mutex + cgroups *simplelru.LRU[model.PathKey, *model.CGroupContext] hostWorkloads *simplelru.LRU[containerutils.CGroupID, *cgroupModel.CacheEntry] containerWorkloads *simplelru.LRU[containerutils.ContainerID, *cgroupModel.CacheEntry] } @@ -80,6 +81,11 @@ func NewResolver() (*Resolver, error) { return nil, err } + cr.cgroups, err = simplelru.NewLRU(2048, func(_ model.PathKey, _ *model.CGroupContext) {}) + if err != nil { + return nil, err + } + return cr, nil } @@ -121,10 +127,19 @@ func (cr *Resolver) AddPID(process *model.ProcessCacheEntry) { } else { cr.hostWorkloads.Add(process.CGroup.CGroupID, newCGroup) } + cr.cgroups.Add(process.CGroup.CGroupFile, &process.CGroup) cr.NotifyListeners(CGroupCreated, newCGroup) } +// GetCGroupContext returns the cgroup context with the specified path key +func (cr *Resolver) GetCGroupContext(cgroupPath model.PathKey) (*model.CGroupContext, bool) { + cr.Lock() + defer cr.Unlock() + + return cr.cgroups.Get(cgroupPath) +} + // GetWorkload returns the workload referenced by the provided ID func (cr *Resolver) GetWorkload(id containerutils.ContainerID) (*cgroupModel.CacheEntry, bool) { if id == "" { @@ -171,6 +186,7 @@ func (cr *Resolver) deleteWorkloadPID(pid uint32, workload *cgroupModel.CacheEnt // check if the workload should be deleted if len(workload.PIDs) <= 0 { + cr.cgroups.Remove(workload.CGroupFile) cr.hostWorkloads.Remove(workload.CGroupID) if workload.ContainerID != "" { cr.containerWorkloads.Remove(workload.ContainerID) @@ -183,5 +199,5 @@ func (cr *Resolver) Len() int { cr.Lock() defer cr.Unlock() - return cr.hostWorkloads.Len() + cr.containerWorkloads.Len() + return cr.cgroups.Len() } diff --git a/pkg/security/resolvers/process/resolver_ebpf.go b/pkg/security/resolvers/process/resolver_ebpf.go index 5bcccdb52c540..409ab86fc2da7 100644 --- a/pkg/security/resolvers/process/resolver_ebpf.go +++ b/pkg/security/resolvers/process/resolver_ebpf.go @@ -340,7 +340,7 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc // Retrieve the container ID of the process from /proc containerID, cgroup, err := p.containerResolver.GetContainerContext(pid) if err != nil { - return fmt.Errorf("snapshot failed for %d: couldn't parse container ID: %w", proc.Pid, err) + return fmt.Errorf("snapshot failed for %d: couldn't parse container and cgroup context: %w", proc.Pid, err) } entry.ContainerID = containerID diff --git a/pkg/security/resolvers/resolvers_ebpf.go b/pkg/security/resolvers/resolvers_ebpf.go index b8899bbd99adf..c9adeb82e557b 100644 --- a/pkg/security/resolvers/resolvers_ebpf.go +++ b/pkg/security/resolvers/resolvers_ebpf.go @@ -219,6 +219,10 @@ func (r *EBPFResolvers) Start(ctx context.Context) error { // ResolveCGroupContext resolves the cgroup context from a cgroup path key func (r *EBPFResolvers) ResolveCGroupContext(pathKey model.PathKey, cgroupFlags containerutils.CGroupFlags) (*model.CGroupContext, error) { + if cgroupContext, found := r.CGroupResolver.GetCGroupContext(pathKey); found { + return cgroupContext, nil + } + path, err := r.DentryResolver.Resolve(pathKey, true) if err != nil { return nil, fmt.Errorf("failed to resolve cgroup file %v: %w", pathKey, err) diff --git a/pkg/security/secl/containerutils/cgroup.go b/pkg/security/secl/containerutils/cgroup.go index 74e7a64540c84..9cc6cca12d884 100644 --- a/pkg/security/secl/containerutils/cgroup.go +++ b/pkg/security/secl/containerutils/cgroup.go @@ -24,6 +24,12 @@ const ( CGroupManagerSystemd // systemd ) +// CGroup flags +const ( + SystemdService CGroupFlags = (0 << 8) + SystemdScope CGroupFlags = (1 << 8) +) + const ( // ContainerRuntimeDocker is used to specify that a container is managed by Docker ContainerRuntimeDocker = "docker" diff --git a/pkg/security/secl/containerutils/helpers.go b/pkg/security/secl/containerutils/helpers.go index 0e46e87af7f2a..311322f94ffc6 100644 --- a/pkg/security/secl/containerutils/helpers.go +++ b/pkg/security/secl/containerutils/helpers.go @@ -29,39 +29,42 @@ func init() { containerIDPattern = regexp.MustCompile(ContainerIDPatternStr) } -func isSystemdCgroup(cgroup CGroupID) bool { - return strings.HasSuffix(string(cgroup), ".service") || strings.HasSuffix(string(cgroup), ".scope") +func isSystemdScope(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".scope") +} + +func isSystemdService(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".service") +} + +func getSystemdCGroupFlags(cgroup CGroupID) uint64 { + if isSystemdScope(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdScope) + } else if isSystemdService(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdService) + } + return 0 } // FindContainerID extracts the first sub string that matches the pattern of a container ID along with the container flags induced from the container runtime prefix func FindContainerID(s CGroupID) (ContainerID, uint64) { match := containerIDPattern.FindIndex([]byte(s)) if match == nil { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - - return "", 0 + return "", getSystemdCGroupFlags(s) } // first, check what's before if match[0] != 0 { previousChar := string(s[match[0]-1]) if strings.ContainsAny(previousChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } // then, check what's after if match[1] < len(s) { nextChar := string(s[match[1]]) if strings.ContainsAny(nextChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } diff --git a/pkg/security/secl/containerutils/helpers_test.go b/pkg/security/secl/containerutils/helpers_test.go index b5474b0df5016..5a2066e0295f2 100644 --- a/pkg/security/secl/containerutils/helpers_test.go +++ b/pkg/security/secl/containerutils/helpers_test.go @@ -63,7 +63,7 @@ func TestFindContainerID(t *testing.T) { { // Some random path which could match garden format input: "/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-f9176c6a-2a34-4ce2-86af-60d16888ed8e.scope", output: "", - flags: CGroupManagerSystemd, + flags: CGroupManagerSystemd | CGroupManager(SystemdScope), }, { // GARDEN with prefix / suffix input: "prefix01234567-0123-4567-890a-bcdesuffix", diff --git a/pkg/security/security_profile/dump/load_controller.go b/pkg/security/security_profile/dump/load_controller.go index ae8767bcfd153..f9bc80cdd3b4f 100644 --- a/pkg/security/security_profile/dump/load_controller.go +++ b/pkg/security/security_profile/dump/load_controller.go @@ -16,6 +16,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/metrics" + "github.com/DataDog/datadog-agent/pkg/security/secl/containerutils" "github.com/DataDog/datadog-agent/pkg/security/secl/model" "github.com/DataDog/datadog-agent/pkg/security/seclog" ) @@ -34,6 +35,7 @@ type ActivityDumpLoadController struct { // eBPF maps activityDumpConfigDefaults *ebpf.Map + activityDumpLoadConfig map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig } // NewActivityDumpLoadController returns a new activity dump load controller @@ -58,7 +60,11 @@ func NewActivityDumpLoadController(adm *ActivityDumpManager) (*ActivityDumpLoadC }, nil } -func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDumpLoadConfig { +func (lc *ActivityDumpLoadController) getDefaultLoadConfigs() (map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig, error) { + if lc.activityDumpLoadConfig != nil { + return lc.activityDumpLoadConfig, nil + } + defaults := NewActivityDumpLoadConfig( lc.adm.config.RuntimeSecurity.ActivityDumpTracedEventTypes, lc.adm.config.RuntimeSecurity.ActivityDumpCgroupDumpTimeout, @@ -68,14 +74,38 @@ func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDump lc.adm.resolvers.TimeResolver, ) defaults.WaitListTimestampRaw = uint64(lc.adm.config.RuntimeSecurity.ActivityDumpCgroupWaitListTimeout) - return defaults + + allDefaultConfigs := map[string]containerutils.CGroupManager{ + containerutils.CGroupManagerDocker.String(): containerutils.CGroupManagerDocker, + containerutils.CGroupManagerPodman.String(): containerutils.CGroupManagerPodman, + containerutils.CGroupManagerCRI.String(): containerutils.CGroupManagerCRI, + containerutils.CGroupManagerCRIO.String(): containerutils.CGroupManagerCRIO, + containerutils.CGroupManagerSystemd.String(): containerutils.CGroupManagerSystemd, + } + defaultConfigs := make(map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig) + for _, cgroupManager := range lc.adm.config.RuntimeSecurity.ActivityDumpCgroupsManagers { + cgroupManager, found := allDefaultConfigs[cgroupManager] + if !found { + return nil, fmt.Errorf("unsupported cgroup manager '%s'", cgroupManager) + } + defaultConfigs[cgroupManager] = defaults + } + lc.activityDumpLoadConfig = defaultConfigs + return defaultConfigs, nil } -// PushCurrentConfig pushes the current load controller config to kernel space -func (lc *ActivityDumpLoadController) PushCurrentConfig() error { +// PushDefaultCurrentConfigs pushes the current load controller configs to kernel space +func (lc *ActivityDumpLoadController) PushDefaultCurrentConfigs() error { + defaultConfigs, err := lc.getDefaultLoadConfigs() + if err != nil { + return err + } + // push default load config values - if err := lc.activityDumpConfigDefaults.Put(uint32(0), lc.getDefaultLoadConfig()); err != nil { - return fmt.Errorf("couldn't update default activity dump load config: %w", err) + for cgroupManager, defaultConfig := range defaultConfigs { + if err := lc.activityDumpConfigDefaults.Put(uint32(cgroupManager), defaultConfig); err != nil { + return fmt.Errorf("couldn't update default activity dump load config: %w", err) + } } return nil } diff --git a/pkg/security/security_profile/dump/manager.go b/pkg/security/security_profile/dump/manager.go index 079f45c13bc21..be81a28ca5c4d 100644 --- a/pkg/security/security_profile/dump/manager.go +++ b/pkg/security/security_profile/dump/manager.go @@ -321,7 +321,8 @@ func NewActivityDumpManager(config *config.Config, statsdClient statsd.ClientInt if err != nil { return nil, fmt.Errorf("couldn't instantiate the activity dump load controller: %w", err) } - if err = loadController.PushCurrentConfig(); err != nil { + + if err = loadController.PushDefaultCurrentConfigs(); err != nil { return nil, fmt.Errorf("failed to push load controller config settings to kernel space: %w", err) } adm.loadController = loadController @@ -449,8 +450,8 @@ func (adm *ActivityDumpManager) HandleCGroupTracingEvent(event *model.CgroupTrac adm.Lock() defer adm.Unlock() - if len(event.ContainerContext.ContainerID) == 0 { - seclog.Warnf("received a cgroup tracing event with an empty container ID") + if len(event.CGroupContext.CGroupID) == 0 { + seclog.Warnf("received a cgroup tracing event with an empty cgroup ID") return } @@ -514,7 +515,19 @@ workloadLoop: } // if we're still here, we can start tracing this workload - if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *adm.loadController.getDefaultLoadConfig()); err != nil { + defaultConfigs, err := adm.loadController.getDefaultLoadConfigs() + if err != nil { + seclog.Errorf("%v", err) + continue + } + + defaultConfig, found := defaultConfigs[containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags)] + if !found { + seclog.Errorf("Failed to find default activity dump config for %s", containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags).String()) + continue + } + + if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *defaultConfig); err != nil { if !errors.Is(err, unix.E2BIG) { seclog.Debugf("%v", err) break diff --git a/pkg/security/utils/cgroup.go b/pkg/security/utils/cgroup.go index 5dbbb97c58af1..df59e317006a5 100644 --- a/pkg/security/utils/cgroup.go +++ b/pkg/security/utils/cgroup.go @@ -12,6 +12,7 @@ import ( "bufio" "bytes" "crypto/sha256" + "fmt" "os" "strconv" "strings" @@ -49,6 +50,49 @@ func (cg ControlGroup) GetContainerID() containerutils.ContainerID { return containerutils.ContainerID(id) } +// GetLastProcControlGroups returns the first cgroup membership of the specified task. +func GetLastProcControlGroups(tgid, pid uint32) (ControlGroup, error) { + data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) + if err != nil { + return ControlGroup{}, err + } + + data = bytes.TrimSpace(data) + + index := bytes.LastIndexByte(data, '\n') + if index < 0 { + index = 0 + } else { + index++ // to skip the \n + } + if index >= len(data) { + return ControlGroup{}, fmt.Errorf("invalid cgroup data: %s", data) + } + + lastLine := string(data[index:]) + + idstr, rest, ok := strings.Cut(lastLine, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + id, err := strconv.Atoi(idstr) + if err != nil { + return ControlGroup{}, err + } + + controllers, path, ok := strings.Cut(rest, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + return ControlGroup{ + ID: id, + Controllers: strings.Split(controllers, ","), + Path: path, + }, nil +} + // GetProcControlGroups returns the cgroup membership of the specified task. func GetProcControlGroups(tgid, pid uint32) ([]ControlGroup, error) { data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) @@ -85,15 +129,14 @@ func GetProcContainerID(tgid, pid uint32) (containerutils.ContainerID, error) { // GetProcContainerContext returns the container ID which the process belongs to along with its manager. Returns "" if the process does not belong // to a container. func GetProcContainerContext(tgid, pid uint32) (containerutils.ContainerID, model.CGroupContext, error) { - cgroups, err := GetProcControlGroups(tgid, pid) - if err != nil || len(cgroups) == 0 { + cgroup, err := GetLastProcControlGroups(tgid, pid) + if err != nil { return "", model.CGroupContext{}, err } - lastCgroup := len(cgroups) - 1 - containerID, runtime := cgroups[lastCgroup].GetContainerContext() + containerID, runtime := cgroup.GetContainerContext() cgroupContext := model.CGroupContext{ - CGroupID: containerutils.CGroupID(cgroups[lastCgroup].Path), + CGroupID: containerutils.CGroupID(cgroup.Path), CGroupFlags: runtime, } diff --git a/pkg/serializer/internal/metrics/service_checks_test.go b/pkg/serializer/internal/metrics/service_checks_test.go index 2a96f3d10072a..2559d7c9ed2cc 100644 --- a/pkg/serializer/internal/metrics/service_checks_test.go +++ b/pkg/serializer/internal/metrics/service_checks_test.go @@ -126,7 +126,7 @@ func TestPayloadsEmptyServiceCheck(t *testing.T) { func TestPayloadsServiceChecks(t *testing.T) { config := mock.New(t) - config.Set("serializer_max_payload_size", 200, pkgconfigmodel.SourceAgentRuntime) + config.Set("serializer_max_payload_size", 250, pkgconfigmodel.SourceAgentRuntime) serviceCheckCollection := []ServiceChecks{ {createServiceCheck("1"), createServiceCheck("2"), createServiceCheck("3")}, diff --git a/pkg/serverless/daemon/routes.go b/pkg/serverless/daemon/routes.go index 733b4a49b050c..fe78651ac4d2f 100644 --- a/pkg/serverless/daemon/routes.go +++ b/pkg/serverless/daemon/routes.go @@ -104,7 +104,17 @@ func (e *EndInvocation) ServeHTTP(w http.ResponseWriter, r *http.Request) { } errorMsg := r.Header.Get(invocationlifecycle.InvocationErrorMsgHeader) + if decodedMsg, err := base64.StdEncoding.DecodeString(errorMsg); err != nil { + log.Debug("Error message header may not be encoded, setting as is") + } else { + errorMsg = string(decodedMsg) + } errorType := r.Header.Get(invocationlifecycle.InvocationErrorTypeHeader) + if decodedType, err := base64.StdEncoding.DecodeString(errorType); err != nil { + log.Debug("Error type header may not be encoded, setting as is") + } else { + errorType = string(decodedType) + } errorStack := r.Header.Get(invocationlifecycle.InvocationErrorStackHeader) if decodedStack, err := base64.StdEncoding.DecodeString(errorStack); err != nil { log.Debug("Could not decode error stack header") diff --git a/pkg/serverless/daemon/routes_test.go b/pkg/serverless/daemon/routes_test.go index 25231204af6c8..e630d2829bd61 100644 --- a/pkg/serverless/daemon/routes_test.go +++ b/pkg/serverless/daemon/routes_test.go @@ -7,6 +7,7 @@ package daemon import ( "bytes" + "encoding/base64" "fmt" "io" "net/http" @@ -104,7 +105,7 @@ func TestEndInvocation(t *testing.T) { assert.Equal(m.lastEndDetails.Runtime, d.ExecutionContext.GetCurrentState().Runtime) } -func TestEndInvocationWithError(t *testing.T) { +func TestEndInvocationWithErrorEncodedHeaders(t *testing.T) { assert := assert.New(t) port := testutil.FreeTCPPort(t) d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) @@ -114,10 +115,52 @@ func TestEndInvocationWithError(t *testing.T) { m := &mockLifecycleProcessor{} d.InvocationProcessor = m + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message \n at TestFunction.Handle(ILambdaContext context)" + + client := &http.Client{} + body := bytes.NewBuffer([]byte(`{}`)) + request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) + request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, base64.StdEncoding.EncodeToString([]byte(errorMessage))) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, base64.StdEncoding.EncodeToString([]byte(errorType))) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, base64.StdEncoding.EncodeToString([]byte(errorStack))) + assert.Nil(err) + res, err := client.Do(request) + assert.Nil(err) + if res != nil { + res.Body.Close() + assert.Equal(res.StatusCode, 200) + } + assert.True(m.OnInvokeEndCalled) + assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) +} + +func TestEndInvocationWithErrorNonEncodedHeaders(t *testing.T) { + assert := assert.New(t) + port := testutil.FreeTCPPort(t) + d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) + time.Sleep(100 * time.Millisecond) + defer d.Stop() + + m := &mockLifecycleProcessor{} + d.InvocationProcessor = m + + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message at TestFunction.Handle(ILambdaContext context)" + client := &http.Client{} body := bytes.NewBuffer([]byte(`{}`)) request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, errorMessage) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, errorType) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, errorStack) assert.Nil(err) res, err := client.Do(request) assert.Nil(err) @@ -127,6 +170,9 @@ func TestEndInvocationWithError(t *testing.T) { } assert.True(m.OnInvokeEndCalled) assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) } func TestTraceContext(t *testing.T) { diff --git a/pkg/serverless/proxy/proxy.go b/pkg/serverless/proxy/proxy.go index 7d43dd36db257..1f8a2e3087658 100644 --- a/pkg/serverless/proxy/proxy.go +++ b/pkg/serverless/proxy/proxy.go @@ -8,6 +8,7 @@ package proxy import ( "context" + "errors" "net/http" "net/http/httputil" "net/url" @@ -68,9 +69,44 @@ func newProxy(target string, processor invocationlifecycle.InvocationProcessor) Scheme: "http", Host: target, } + proxy := httputil.NewSingleHostReverseProxy(url) + + // The default error handler logs "http: proxy error: %v" then returns an HTTP 502 (bad gateway) + // response. This is unfortunate because it lacks much any context on the original request that + // failed, and the commonly observed error today is "context deadline exceeded", which is not + // actionnable if you don't know what request it was for. It also logs to STDERR and does not + // honor the agent's log level. + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + log.Debugf( + "[serverless/proxy][%T] %s %s -- proxy error: %v", + // The dynamic type of processor informs about what kind of proxy this was (main/appsec) + processor, + // The request method and URL are useful to understand what exactly failed. We won't log + // the body (too large) or headers (risks containing sensitive data, such as API keys) + r.Method, r.URL, + // What happened that caused us to be called? + err, + ) + + // If the error is a [context.DeadlineExceeded], we return an HTTP 504 (gateway timeout) + // instead of the generic HTTP 502 (bad gateway) to give the client a better idea of what is + // going on (this may influence retry behavior, for example). + if errors.Is(err, context.DeadlineExceeded) { + w.WriteHeader(http.StatusGatewayTimeout) + } else { + // Return an HTTP 502 (bad gateway) error response; defer the retrying to the client. + w.WriteHeader(http.StatusBadGateway) + } + + // Writing the error message as best-effort, we simply debug-log any error that occur here. + if _, err := w.Write([]byte(err.Error())); err != nil { + log.Debugf("[serverless/proxy][%T] failed to write error message to response body: %v", processor, err) + } + } + return &runtimeProxy{ target: url, - proxy: httputil.NewSingleHostReverseProxy(url), + proxy: proxy, processor: processor, } } diff --git a/pkg/trace/api/api.go b/pkg/trace/api/api.go index c74ecbb137fb9..5cb57d2478a27 100644 --- a/pkg/trace/api/api.go +++ b/pkg/trace/api/api.go @@ -141,7 +141,7 @@ func NewHTTPReceiver( } } log.Infof("Receiver configured with %d decoders and a timeout of %dms", semcount, conf.DecoderTimeout) - containerIDProvider := NewIDProvider(conf.ContainerProcRoot) + containerIDProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) telemetryForwarder := NewTelemetryForwarder(conf, containerIDProvider, statsd) return &HTTPReceiver{ Stats: info.NewReceiverStats(), diff --git a/pkg/trace/api/api_test.go b/pkg/trace/api/api_test.go index 9a468dc164487..2a1bdafe71307 100644 --- a/pkg/trace/api/api_test.go +++ b/pkg/trace/api/api_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/trace/config" @@ -576,7 +577,9 @@ func TestDecodeV05(t *testing.T) { req, err := http.NewRequest("POST", "/v0.5/traces", bytes.NewReader(b)) assert.NoError(err) req.Header.Set(header.ContainerID, "abcdef123789456") - tp, err := decodeTracerPayload(v05, req, NewIDProvider(""), "python", "3.8.1", "1.2.3") + tp, err := decodeTracerPayload(v05, req, NewIDProvider("", func(_ origindetection.OriginInfo) (string, error) { + return "abcdef123789456", nil + }), "python", "3.8.1", "1.2.3") assert.NoError(err) assert.EqualValues(tp, &pb.TracerPayload{ ContainerID: "abcdef123789456", diff --git a/pkg/trace/api/container.go b/pkg/trace/api/container.go index 508277c5abaf9..0c789f9d06ca7 100644 --- a/pkg/trace/api/container.go +++ b/pkg/trace/api/container.go @@ -12,6 +12,7 @@ import ( "net" "net/http" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" ) @@ -28,7 +29,7 @@ type IDProvider interface { type idProvider struct{} // NewIDProvider initializes an IDProvider instance, in non-linux environments the procRoot arg is unused. -func NewIDProvider(_ string) IDProvider { +func NewIDProvider(_ string, _ func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { return &idProvider{} } diff --git a/pkg/trace/api/container_linux.go b/pkg/trace/api/container_linux.go index 2129a6377c97f..6dd12d227894d 100644 --- a/pkg/trace/api/container_linux.go +++ b/pkg/trace/api/container_linux.go @@ -18,6 +18,7 @@ import ( "syscall" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/util/cgroups" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -96,7 +97,7 @@ func (i *noCgroupsProvider) GetContainerID(_ context.Context, h http.Header) str } // NewIDProvider initializes an IDProvider instance using the provided procRoot to perform cgroups lookups in linux environments. -func NewIDProvider(procRoot string) IDProvider { +func NewIDProvider(procRoot string, containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { // taken from pkg/util/containers/metrics/system.collector_linux.go var hostPrefix string if strings.HasPrefix(procRoot, "/host") { @@ -120,10 +121,11 @@ func NewIDProvider(procRoot string) IDProvider { } c := NewCache(1 * time.Minute) return &cgroupIDProvider{ - procRoot: procRoot, - controller: cgroupController, - cache: c, - reader: reader, + procRoot: procRoot, + controller: cgroupController, + cache: c, + reader: reader, + containerIDFromOriginInfo: containerIDFromOriginInfo, } } @@ -131,8 +133,9 @@ type cgroupIDProvider struct { procRoot string controller string // reader is used to retrieve the container ID from its cgroup v2 inode. - reader *cgroups.Reader - cache *Cache + reader *cgroups.Reader + cache *Cache + containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) } // GetContainerID returns the container ID. @@ -157,6 +160,11 @@ func (c *cgroupIDProvider) GetContainerID(ctx context.Context, h http.Header) st return containerID } + // Retrieve container ID from External Data header + if externalData := h.Get(header.ExternalData); externalData != "" { + return c.resolveContainerIDFromExternalData(externalData) + } + return "" } @@ -296,6 +304,27 @@ func (c *cgroupIDProvider) getCachedContainerID(key string, retrievalFunc func() return val, nil } +// resolveContainerIDFromExternalData returns the container ID for the given External Data. +func (c *cgroupIDProvider) resolveContainerIDFromExternalData(rawExternalData string) string { + var generatedContainerID string + + externalData, err := origindetection.ParseExternalData(rawExternalData) + if err != nil { + log.Errorf("Could not parse external data (%s): %v", rawExternalData, err) + return "" + } + generatedContainerID, err = c.containerIDFromOriginInfo(origindetection.OriginInfo{ + ExternalData: externalData, + ProductOrigin: origindetection.ProductOriginAPM, + }) + if err != nil { + log.Errorf("Could not generate container ID from external data (%s): %v", rawExternalData, err) + return "" + } + + return generatedContainerID +} + // The below cache is copied from /pkg/util/containers/v2/metrics/provider/cache.go. It is not // imported to avoid making the datadog-agent module a dependency of the pkg/trace module. The // datadog-agent module contains replace directives which are not inherited by packages that diff --git a/pkg/trace/api/debugger.go b/pkg/trace/api/debugger.go index 61164f1d3ed02..51816276a3f3b 100644 --- a/pkg/trace/api/debugger.go +++ b/pkg/trace/api/debugger.go @@ -81,7 +81,7 @@ func debuggerErrorHandler(err error) http.Handler { // newDebuggerProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newDebuggerProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/api/evp_proxy.go b/pkg/trace/api/evp_proxy.go index 452d73017aa8d..2795e6304b605 100644 --- a/pkg/trace/api/evp_proxy.go +++ b/pkg/trace/api/evp_proxy.go @@ -86,7 +86,7 @@ func evpProxyForwarder(conf *config.AgentConfig, statsd statsd.ClientInterface) req.Header["X-Forwarded-For"] = nil }, ErrorLog: logger, - Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot), statsd}, + Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo), statsd}, } } diff --git a/pkg/trace/api/internal/header/headers.go b/pkg/trace/api/internal/header/headers.go index 8ae458be834f5..88cb3549b2836 100644 --- a/pkg/trace/api/internal/header/headers.go +++ b/pkg/trace/api/internal/header/headers.go @@ -25,6 +25,16 @@ const ( // * "ci-,in-" LocalData = "Datadog-Entity-ID" + // ExternalData is a list that contain prefixed-items, split by a ','. Current items are: + // * "it-" if the container is an init container. + // * "cn-" for the container name. + // * "pu-" for the pod UID. + // Order does not matter. + // Possible values: + // * "it-false,cn-nginx,pu-3413883c-ac60-44ab-96e0-9e52e4e173e2" + // * "cn-init,pu-cb4aba1d-0129-44f1-9f1b-b4dc5d29a3b3,it-true" + ExternalData = "Datadog-External-Env" + // Lang specifies the name of the header which contains the language from // which the traces originate. Lang = "Datadog-Meta-Lang" diff --git a/pkg/trace/api/otlp.go b/pkg/trace/api/otlp.go index 88491ea9052ec..eb136d267e090 100644 --- a/pkg/trace/api/otlp.go +++ b/pkg/trace/api/otlp.go @@ -9,7 +9,6 @@ import ( "context" "encoding/hex" "fmt" - "github.com/DataDog/datadog-agent/pkg/trace/transform" "math" "net" "net/http" @@ -26,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/trace/sampler" "github.com/DataDog/datadog-agent/pkg/trace/timing" "github.com/DataDog/datadog-agent/pkg/trace/traceutil" + "github.com/DataDog/datadog-agent/pkg/trace/transform" "github.com/DataDog/datadog-go/v5/statsd" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" @@ -102,7 +102,7 @@ func NewOTLPReceiver(out chan<- *Payload, cfg *config.AgentConfig, statsd statsd enableReceiveResourceSpansV2Val = 1.0 } _ = statsd.Gauge("datadog.trace_agent.otlp.enable_receive_resource_spans_v2", enableReceiveResourceSpansV2Val, nil, 1) - return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} + return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} } // Start starts the OTLPReceiver, if any of the servers were configured as active. diff --git a/pkg/trace/api/pipeline_stats.go b/pkg/trace/api/pipeline_stats.go index b6fe3f2cd161c..b8daf12590bf7 100644 --- a/pkg/trace/api/pipeline_stats.go +++ b/pkg/trace/api/pipeline_stats.go @@ -71,7 +71,7 @@ func pipelineStatsErrorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. func newPipelineStatsProxy(conf *config.AgentConfig, urls []*url.URL, apiKeys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { log.Debug("[pipeline_stats] Creating reverse proxy") - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/profiles.go b/pkg/trace/api/profiles.go index 9b43efc94de74..19add3e307daa 100644 --- a/pkg/trace/api/profiles.go +++ b/pkg/trace/api/profiles.go @@ -109,7 +109,7 @@ func errorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. // For more details please see multiTransport. func newProfileProxy(conf *config.AgentConfig, targets []*url.URL, keys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/symdb.go b/pkg/trace/api/symdb.go index 31b96c494ce5f..b238b478eb65c 100644 --- a/pkg/trace/api/symdb.go +++ b/pkg/trace/api/symdb.go @@ -62,7 +62,7 @@ func symDBErrorHandler(err error) http.Handler { // newSymDBProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newSymDBProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getSymDBDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/config/config.go b/pkg/trace/config/config.go index 5d0e59f059e39..cbf0b0ea5c3a3 100644 --- a/pkg/trace/config/config.go +++ b/pkg/trace/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/obfuscate" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/trace/log" @@ -445,6 +446,9 @@ type AgentConfig struct { // ContainerTags ... ContainerTags func(cid string) ([]string, error) `json:"-"` + // ContainerIDFromOriginInfo ... + ContainerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) `json:"-"` + // ContainerProcRoot is the root dir for `proc` info ContainerProcRoot string diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index 51564f328f09e..29bf241afd7b8 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -52,6 +52,7 @@ require ( ) require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/shirou/gopsutil/v4 v4.24.11 go.opentelemetry.io/collector/component/componenttest v0.115.0 ) @@ -115,6 +116,7 @@ require ( replace ( github.com/DataDog/datadog-agent => ../../ + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../comp/trace/compression/impl-gzip github.com/DataDog/datadog-agent/comp/trace/compression/impl-zstd => ../../comp/trace/compression/impl-zstd diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index c3759db9cc74b..071e7c2c2eba0 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -20,6 +20,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect @@ -87,6 +88,7 @@ require ( ) replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../../../../comp/otelcol/otlp/components/metricsclient github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ../../../../comp/otelcol/otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../comp/trace/compression/def diff --git a/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml new file mode 100644 index 0000000000000..4aa3e9ef23e70 --- /dev/null +++ b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Implement External Data resolution for APM. This is needed to support the + latest Origin Detection spec and resolution with nested virtualization. + diff --git a/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml new file mode 100644 index 0000000000000..7d9a8653d8f0d --- /dev/null +++ b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Metric payloads are compressed using `zstd` compression by default. + This can be reverted to the previous compression kind by adding + ``serializer_compressor_kind: zlib`` to the configuration. diff --git a/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml new file mode 100644 index 0000000000000..2957ae7f9441d --- /dev/null +++ b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Adds support for the `none` cardinality type in the tagger. diff --git a/tasks/buildimages.py b/tasks/buildimages.py index e2bf5316e18c7..06ea84211ac1a 100644 --- a/tasks/buildimages.py +++ b/tasks/buildimages.py @@ -46,13 +46,23 @@ def update_test_infra_definitions(ctx: Context, commit_sha: str, go_mod_only: bo """ Update the test-infra-definition image version in the Gitlab CI as well as in the e2e go.mod """ - if not go_mod_only: - update_test_infra_def(".gitlab/common/test_infra_version.yml", commit_sha[:12], is_dev_image) - + print(f"Updating test-infra-definitions to {commit_sha}") with ctx.cd("test/new-e2e"): ctx.run(f"go get github.com/DataDog/test-infra-definitions@{commit_sha}") ctx.run("go mod tidy") + if not go_mod_only: + prefix_comment = """# File generated by inv buildimages.update-test-infra-definitions +# Please do not edit this file manually +# To update the test-infra-definitions version, run `inv buildimages.update-test-infra-definitions --commit-sha ` [--is-dev-image] +""" + update_test_infra_def( + file_path=".gitlab/common/test_infra_version.yml", + image_tag=commit_sha[:12], + is_dev_image=is_dev_image, + prefix_comment=prefix_comment, + ) + @task( help={ diff --git a/tasks/libs/ciproviders/gitlab_api.py b/tasks/libs/ciproviders/gitlab_api.py index adfd8aacfb569..f1c0b4784d2f6 100644 --- a/tasks/libs/ciproviders/gitlab_api.py +++ b/tasks/libs/ciproviders/gitlab_api.py @@ -1258,12 +1258,10 @@ def full_config_get_all_stages(full_config: dict) -> set[str]: return all_stages -def update_test_infra_def(file_path, image_tag, is_dev_image=False): +def update_test_infra_def(file_path, image_tag, is_dev_image=False, prefix_comment=""): """ Updates TEST_INFRA_DEFINITIONS_BUILDIMAGES in `.gitlab/common/test_infra_version.yml` file """ - import yaml - test_infra_def = {} with open(file_path) as test_infra_version_file: try: @@ -1276,6 +1274,7 @@ def update_test_infra_def(file_path, image_tag, is_dev_image=False): except yaml.YAMLError as e: raise Exit(f"Error while loading {file_path}: {e}") from e with open(file_path, "w") as test_infra_version_file: + test_infra_version_file.write(prefix_comment + ('\n\n' if prefix_comment else '')) # Add explicit_start=True to keep the document start marker --- # See "Document Start" in https://www.yaml.info/learn/document.html for more details yaml.dump(test_infra_def, test_infra_version_file, explicit_start=True) diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 92faae31be87d..be219da1e3d81 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -58,7 +58,7 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa + github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 @@ -72,10 +72,10 @@ require ( github.com/kr/pretty v0.3.1 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c github.com/pkg/sftp v1.13.7 - github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 - github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 - github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 - github.com/pulumi/pulumi/sdk/v3 v3.140.0 + github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 + github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 + github.com/pulumi/pulumi/sdk/v3 v3.142.0 github.com/samber/lo v1.47.0 github.com/stretchr/testify v1.10.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -171,7 +171,7 @@ require ( github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl/v2 v2.20.1 // indirect + github.com/hashicorp/hcl/v2 v2.22.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index e922c7c7a56b9..7284ab1424a36 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -17,8 +17,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa h1:l8KLWgU9l2qTlMtu4ing3V6PptTO+suaU8zusc45IiM= -github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa/go.mod h1:1PAUwGjC25ACjfft4HrLEmHliuajlvjzcLFWpuqAIyk= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 h1:w+uoG7RdtPiW6YvJvWrzWOC78eNzCemcH8ZM6tJoUBw= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266/go.mod h1:+13pRFKChJo9VZ0WcHFm5GTPCNYyim3hxFjy6cpcLG8= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= @@ -268,8 +268,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.22.0 h1:hkZ3nCtqeJsDhPRFz5EA9iwcG1hNWGePOTw6oyul12M= +github.com/hashicorp/hcl/v2 v2.22.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -403,10 +403,10 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435 github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE= github.com/pulumi/esc v0.10.0 h1:jzBKzkLVW0mePeanDRfqSQoCJ5yrkux0jIwAkUxpRKE= github.com/pulumi/esc v0.10.0/go.mod h1:2Bfa+FWj/xl8CKqRTWbWgDX0SOD4opdQgvYSURTGK2c= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 h1:wA38Ep4sEphX+3YGwFfaxRHs7NQv8dNObFepX6jaRa4= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1/go.mod h1:m/ejZ2INurqq/ncDjJfgC1Ff/lnbt0J/uO33BnPVots= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 h1:6082hB+ILpPB/0V5F+LTmHbX1BO54tCVOQCVOL/FYI4= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1/go.mod h1:z2bnBPHNYfk72IW1P01H9qikBtBSBhCwi3QpH6Y/38Q= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 h1:OvCLqUueOja9YE2WEGPYAw+lKHFRbLQ7QjwX55+uNsA= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0/go.mod h1:FFzye44v9E0BgaFXVB/9X7KH0S0MapoXEy2YonrQfz4= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 h1:jil2EBzZnKsRDrLfvx2gnAaq17HQLrTbpPsIb3h+98U= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0/go.mod h1:r+K4M7jnLqvvQDeR/0mBRq2EPZaqsDg24Ciy3ml/thA= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1 h1:miIJy4njnFYw7VxMLvEztoMPr9zYC2kqBTwRlaFAf48= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1/go.mod h1:LR1QBq0C1NIhmD9E0uKozCAu32j5qsamhrIsTSNVMS8= github.com/pulumi/pulumi-azure-native-sdk/compute/v2 v2.73.1 h1:79HTKSE1uJQolCRUHRFnIbSPNSIhxekIhznHnjpLi6s= @@ -427,16 +427,16 @@ github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 h1:s2Cpu6E2lmADNUbutbJGm6O+O9j0mBLlrh github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 h1:21oSj+TKlKTzQcxN9Hik7iSNNHPUQXN4s3itOnahy/w= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0/go.mod h1:YaEZms1NgXFqGhObKVofcAeWXu2V+3t/BAXdHQZq7fU= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 h1:quqoGsLbF7lpGpGU4mi5WfVLIAo4gfvoQeYYmemx1Dg= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3/go.mod h1:9dBA6+rtpKmyZB3k1XryUOHDOuNdoTODFKEEZZCtrz8= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 h1:7AjJpUyW6YHHpZr0bI6Fy1A3/b7ERxq1LAo5mlyNN1Y= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0/go.mod h1:ATS+UN8pguMxypQAK+SaPewesU+UN5dpf93PNqVuHzs= github.com/pulumi/pulumi-libvirt/sdk v0.5.3 h1:CiUGTweLLIxbAbADxxnwPv4BK8pxXfU8urokJvK1ihM= github.com/pulumi/pulumi-libvirt/sdk v0.5.3/go.mod h1:gAhyIZKtzs4rknrl8fu8BQnyqijAmViFbaUkyuHt4xY= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7 h1:39rhOe/PTUGMYia8pR5T2wbxxMt2pwrlonf0ncYKSzE= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7/go.mod h1:cxxDhJzUPt/YElfvlWa15Q4NGF6XXS8kUs4OQsCxSBk= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 h1:tXemWrzeVTqG8zq6hBdv1TdPFXjgZ+dob63a/6GlF1o= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1/go.mod h1:hODo3iEmmXDFOXqPK+V+vwI0a3Ww7BLjs5Tgamp86Ng= -github.com/pulumi/pulumi/sdk/v3 v3.140.0 h1:+Z/RBvdYg7tBNkBwk4p/FzlV7niBT3TbLAICq/Y0LDU= -github.com/pulumi/pulumi/sdk/v3 v3.140.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= +github.com/pulumi/pulumi/sdk/v3 v3.142.0 h1:SmcVddGuvwAh3g3XUVQQ5gVRQUKH1yZ6iETpDNHIHlw= +github.com/pulumi/pulumi/sdk/v3 v3.142.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= github.com/pulumiverse/pulumi-time/sdk v0.1.0 h1:xfi9HKDgV+GgDxQ23oSv9KxC3DQqViGTcMrJICRgJv0= github.com/pulumiverse/pulumi-time/sdk v0.1.0/go.mod h1:NUa1zA74DF002WrM6iF111A6UjX9knPpXufVRvBwNyg= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= diff --git a/test/otel/go.mod b/test/otel/go.mod index a4523c0b33b0b..88d5f3a127be5 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ./../../comp/core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ./../../comp/core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../comp/core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ./../../comp/core/telemetry github.com/DataDog/datadog-agent/comp/def => ./../../comp/def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../comp/forwarder/defaultforwarder @@ -129,6 +130,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect diff --git a/test/regression/config.yaml b/test/regression/config.yaml index 362301357f864..4dcef653249aa 100644 --- a/test/regression/config.yaml +++ b/test/regression/config.yaml @@ -1,5 +1,5 @@ lading: - version: 0.24.0 + version: 0.25.2 target: cpu_allotment: 8