Merge branch 'main' into daniel.lavie/USMON-830-kafka-fetch-latency

.github/CODEOWNERS

@@ -17,6 +17,9 @@
 # changing the config of mockery will regenerate the mocks, so owners will have to review anyway
 /.mockery.yaml                          # do not notify anyone
 /.go-version                            @DataDog/agent-shared-components @DataDog/agent-delivery
+# Go linters and pre-commit config
+/.golangci.yml                          @DataDog/agent-devx-loops
+/.pre-commit-config.yaml                @DataDog/agent-devx-loops
 
 /CHANGELOG.rst                          @DataDog/agent-delivery
 /CHANGELOG-DCA.rst                      @DataDog/container-integrations @DataDog/container-platform
@@ -30,14 +33,14 @@
 # Todo: is this file still needed?
 /Makefile.trace                         @DataDog/agent-delivery
 
+/mkdocs.yml                             @DataDog/agent-devx-infra
 /release.json                           @DataDog/agent-delivery @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations @DataDog/agent-release-management @DataDog/agent-security
 /requirements.txt                       @DataDog/agent-devx-infra
 /pyproject.toml                         @DataDog/agent-devx-infra @DataDog/agent-devx-loops
-/setup.cfg                              @DataDog/agent-devx-infra
 /repository.datadog.yml                 @DataDog/agent-devx-infra
 /generate_tools.go                      @DataDog/agent-devx-infra
 /service.datadog.yaml                   @DataDog/agent-delivery
-/.pre-commit-config.yaml                @DataDog/agent-devx-loops
+/static-analysis.datadog.yaml           @DataDog/software-integrity-and-trust @DataDog/agent-devx-infra
 
 /.circleci/                             @DataDog/agent-devx-infra
 

.github/labeler.yml

@@ -4,6 +4,7 @@ component/system-probe:
 - changed-files:
   - any-glob-to-any-file:
     - pkg/collector/corechecks/ebpf/** #ebpf-platform (oomkill and tcp_queue_length)
+    - pkg/collector/corechecks/servicediscovery/module/** # usm
     - pkg/ebpf/** # ebpf-platform (ebpf_manager)
     - pkg/eventmonitor/** # cws (new event monitor component)
     - pkg/network/** # npm and usm

.gitlab-ci.yml

@@ -736,6 +736,7 @@ workflow:
 
 .system_probe_change_paths: &system_probe_change_paths
   - pkg/collector/corechecks/ebpf/**/*
+  - pkg/collector/corechecks/servicediscovery/module/*
   - pkg/ebpf/**/*
   - pkg/network/**/*
   - pkg/process/monitor/*

.golangci.yml

@@ -5,12 +5,12 @@ issues:
 
   # Do not limit the number of times a same issue is reported.
   max-same-issues: 0
-  
+
   exclude-files:
     - pkg/util/cloudproviders/cloudfoundry/bbscache_test.go # implements interface from imported package whose method names fail linting
     - pkg/util/intern/string.go # TODO: fix govet 'unsafeptr' error
     - pkg/serverless/trace/inferredspan/constants.go # TODO: fox revive exported const error
-    
+
   exclude-dirs:
     - pkg/proto/patches
     - tasks/unit_tests/testdata/components_src
@@ -122,11 +122,11 @@ linters-settings:
   revive:
     # in order to change revive config, the default configuration must be explicitly defined
     # https://github.com/mgechev/revive/blob/master/defaults.toml
-    ignoreGeneratedHeader: false
+    #
+    # see https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml for the golangci-lint config syntax
+    ignore-generated-header: false
     severity: "warning"
     confidence: 0.8
-    errorCode: 0
-    warningCode: 0
     rules:
       - name: blank-imports
       - name: context-as-argument

cmd/cluster-agent/subcommands/start/command.go

@@ -347,22 +347,25 @@ func start(log log.Component,
 	var rcClient *rcclient.Client
 	rcserv, isSet := rcService.Get()
 	if pkgconfig.IsRemoteConfigEnabled(config) && isSet {
-		// TODO: Is APM Tracing always required? I am not sure
-		products := []string{state.ProductAPMTracing}
-
+		var products []string
+		if config.GetBool("admission_controller.auto_instrumentation.patcher.enabled") {
+			products = append(products, state.ProductAPMTracing)
+		}
 		if config.GetBool("autoscaling.workload.enabled") {
 			products = append(products, state.ProductContainerAutoscalingSettings, state.ProductContainerAutoscalingValues)
 		}
 
-		var err error
-		rcClient, err = initializeRemoteConfigClient(rcserv, config, clusterName, clusterID, products...)
-		if err != nil {
-			log.Errorf("Failed to start remote-configuration: %v", err)
-		} else {
-			rcClient.Start()
-			defer func() {
-				rcClient.Close()
-			}()
+		if len(products) > 0 {
+			var err error
+			rcClient, err = initializeRemoteConfigClient(rcserv, config, clusterName, clusterID, products...)
+			if err != nil {
+				log.Errorf("Failed to start remote-configuration: %v", err)
+			} else {
+				rcClient.Start()
+				defer func() {
+					rcClient.Close()
+				}()
+			}
 		}
 	}
 
@@ -421,7 +424,7 @@ func start(log log.Component,
 			log.Error("Admission controller is disabled, vertical autoscaling requires the admission controller to be enabled. Vertical scaling will be disabled.")
 		}
 
-		if adapter, err := workload.StartWorkloadAutoscaling(mainCtx, apiCl, rcClient, wmeta); err != nil {
+		if adapter, err := workload.StartWorkloadAutoscaling(mainCtx, clusterID, apiCl, rcClient, wmeta, demultiplexer); err != nil {
 			pkglog.Errorf("Error while starting workload autoscaling: %v", err)
 		} else {
 			pa = adapter

comp/core/log/impl-trace/go.mod

@@ -135,6 +135,7 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect
 	google.golang.org/grpc v1.64.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

comp/core/tagger/taggerimpl/remote/tagger.go

@@ -79,10 +79,10 @@ func NodeAgentOptions(config configComponent.Component) (Options, error) {
 	}, nil
 }
 
-// NodeAgentOptionsForSecruityResolvers is a legacy function that returns the
+// NodeAgentOptionsForSecurityResolvers is a legacy function that returns the
 // same options as NodeAgentOptions, but it's used by the tag security resolvers only
 // TODO (component): remove this function once the security resolver migrates to component
-func NodeAgentOptionsForSecruityResolvers() (Options, error) {
+func NodeAgentOptionsForSecurityResolvers() (Options, error) {
 	return Options{
 		Target:       fmt.Sprintf(":%v", config.Datadog().GetInt("cmd_port")),
 		TokenFetcher: func() (string, error) { return security.FetchAuthToken(config.Datadog()) },

comp/trace/agent/impl/agent.go

@@ -15,6 +15,7 @@ import (
 	"os"
 	"os/signal"
 	"runtime"
+	"runtime/debug"
 	"runtime/pprof"
 	"strconv"
 	"sync"
@@ -29,8 +30,10 @@ import (
 	traceagent "github.com/DataDog/datadog-agent/comp/trace/agent/def"
 	compression "github.com/DataDog/datadog-agent/comp/trace/compression/def"
 	"github.com/DataDog/datadog-agent/comp/trace/config"
+	coreconfig "github.com/DataDog/datadog-agent/pkg/config"
 	"github.com/DataDog/datadog-agent/pkg/pidfile"
 	pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace"
+	agentrt "github.com/DataDog/datadog-agent/pkg/runtime"
 	pkgagent "github.com/DataDog/datadog-agent/pkg/trace/agent"
 	tracecfg "github.com/DataDog/datadog-agent/pkg/trace/config"
 	"github.com/DataDog/datadog-agent/pkg/trace/telemetry"
@@ -118,6 +121,9 @@ func NewAgent(deps dependencies) (traceagent.Component, error) {
 		return nil, err
 	}
 	setupShutdown(ctx, deps.Shutdowner, statsdCl)
+
+	prepGoRuntime(tracecfg)
+
 	c.Agent = pkgagent.NewAgent(
 		ctx,
 		c.config.Object(),
@@ -135,6 +141,49 @@ func NewAgent(deps dependencies) (traceagent.Component, error) {
 	return c, nil
 }
 
+func prepGoRuntime(tracecfg *tracecfg.AgentConfig) {
+	cgsetprocs := agentrt.SetMaxProcs()
+	if !cgsetprocs {
+		if mp, ok := os.LookupEnv("GOMAXPROCS"); ok {
+			log.Infof("GOMAXPROCS manually set to %v", mp)
+		} else if tracecfg.MaxCPU > 0 {
+			allowedCores := int(tracecfg.MaxCPU / 100)
+			if allowedCores < 1 {
+				allowedCores = 1
+			}
+			if allowedCores < runtime.GOMAXPROCS(0) {
+				log.Infof("apm_config.max_cpu is less than current GOMAXPROCS. Setting GOMAXPROCS to (%v) %d\n", allowedCores, allowedCores)
+				runtime.GOMAXPROCS(int(allowedCores))
+			}
+		} else {
+			log.Infof("apm_config.max_cpu is disabled. leaving GOMAXPROCS at current value.")
+		}
+	}
+	log.Infof("Trace Agent final GOMAXPROCS: %v", runtime.GOMAXPROCS(0))
+
+	cgmem, err := agentrt.SetGoMemLimit(coreconfig.IsContainerized())
+	if err != nil {
+		log.Infof("Couldn't set Go memory limit from cgroup: %s", err)
+	}
+	if cgmem == 0 {
+		// memory limit not set from cgroups
+		if lim, ok := os.LookupEnv("GOMEMLIMIT"); ok {
+			log.Infof("GOMEMLIMIT manually set to: %v", lim)
+		} else if tracecfg.MaxMemory > 0 {
+			// We have apm_config.max_memory, and no cgroup memory limit is in place.
+			// log.Infof("apm_config.max_memory: %vMiB", int64(tracecfg.MaxMemory)/(1024*1024))
+			finalmem := int64(tracecfg.MaxMemory * 0.9)
+			debug.SetMemoryLimit(finalmem)
+			log.Infof("apm_config.max_memory set to: %vMiB. Setting GOMEMLIMIT to 90%% of max: %vMiB", int64(tracecfg.MaxMemory)/(1024*1024), finalmem/(1024*1024))
+		} else {
+			// There are no memory constraints
+			log.Infof("GOMEMLIMIT unconstrained.")
+		}
+	} else {
+		log.Infof("Memory constrained by cgroup. GOMEMLIMIT is: %vMiB", cgmem/(1024*1024))
+	}
+}
+
 func start(ag component) error {
 	if ag.params.CPUProfile != "" {
 		f, err := os.Create(ag.params.CPUProfile)

comp/trace/agent/impl/run.go

@@ -9,9 +9,6 @@ import (
 	"fmt"
 	"math/rand"
 	"net/http"
-	"os"
-	"runtime"
-	"runtime/debug"
 	"time"
 
 	remotecfg "github.com/DataDog/datadog-agent/cmd/trace-agent/config/remote"
@@ -20,7 +17,6 @@ import (
 	apiutil "github.com/DataDog/datadog-agent/pkg/api/util"
 	coreconfig "github.com/DataDog/datadog-agent/pkg/config"
 	rc "github.com/DataDog/datadog-agent/pkg/config/remote/client"
-	agentrt "github.com/DataDog/datadog-agent/pkg/runtime"
 	"github.com/DataDog/datadog-agent/pkg/trace/api"
 	tracecfg "github.com/DataDog/datadog-agent/pkg/trace/config"
 	"github.com/DataDog/datadog-agent/pkg/trace/info"
@@ -82,49 +78,6 @@ func runAgentSidekicks(ag component) error {
 		},
 	})
 
-	// prepare go runtime
-	cgsetprocs := agentrt.SetMaxProcs()
-	if !cgsetprocs {
-		if mp, ok := os.LookupEnv("GOMAXPROCS"); ok {
-			log.Infof("GOMAXPROCS manually set to %v", mp)
-		} else if tracecfg.MaxCPU > 0 {
-			allowedCores := int(tracecfg.MaxCPU / 100)
-			if allowedCores < 1 {
-				allowedCores = 1
-			}
-			if allowedCores < runtime.GOMAXPROCS(0) {
-				log.Infof("apm_config.max_cpu is less than current GOMAXPROCS. Setting GOMAXPROCS to (%v) %d\n", allowedCores, (allowedCores))
-				runtime.GOMAXPROCS(int(allowedCores))
-			}
-		} else {
-			log.Infof("apm_config.max_cpu is disabled. leaving GOMAXPROCS at current value.")
-		}
-	}
-	log.Infof("Trace Agent final GOMAXPROCS: %v", runtime.GOMAXPROCS(0))
-
-	// prepare go runtime
-	cgmem, err := agentrt.SetGoMemLimit(coreconfig.IsContainerized())
-	if err != nil {
-		log.Infof("Couldn't set Go memory limit from cgroup: %s", err)
-	}
-	if cgmem == 0 {
-		// memory limit not set from cgroups
-		if lim, ok := os.LookupEnv("GOMEMLIMIT"); ok {
-			log.Infof("GOMEMLIMIT manually set to: %v", lim)
-		} else if tracecfg.MaxMemory > 0 {
-			// We have apm_config.max_memory, and no cgroup memory limit is in place.
-			// log.Infof("apm_config.max_memory: %vMiB", int64(tracecfg.MaxMemory)/(1024*1024))
-			finalmem := int64(tracecfg.MaxMemory * 0.9)
-			debug.SetMemoryLimit(finalmem)
-			log.Infof("apm_config.max_memory set to: %vMiB. Setting GOMEMLIMIT to 90%% of max: %vMiB", int64(tracecfg.MaxMemory)/(1024*1024), finalmem/(1024*1024))
-		} else {
-			// There are no memory constraints
-			log.Infof("GOMEMLIMIT unconstrained.")
-		}
-	} else {
-		log.Infof("Memory constrained by cgroup. GOMEMLIMIT is: %vMiB", cgmem/(1024*1024))
-	}
-
 	log.Infof("Trace agent running on host %s", tracecfg.Hostname)
 	if pcfg := profilingConfig(tracecfg); pcfg != nil {
 		if err := profiling.Start(*pcfg); err != nil {