Merge branch 'main' into guillaume.pagnoux/USMON-1365-wrap-config-new

.circleci/config.yml

@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: gcr.io/datadoghq/agent-circleci-runner:v48372186-ff395e52
+      - image: gcr.io/datadoghq/agent-circleci-runner:v50094600-7b86dd80
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent

.github/CODEOWNERS

@@ -36,7 +36,7 @@
 /Makefile.trace                         @DataDog/agent-delivery
 
 /mkdocs.yml                             @DataDog/agent-devx-infra
-/release.json                           @DataDog/agent-delivery @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations @DataDog/agent-release-management @DataDog/agent-security
+/release.json                           @DataDog/agent-delivery @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations @DataDog/agent-security
 /requirements.txt                       @DataDog/agent-devx-infra
 /pyproject.toml                         @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /repository.datadog.yml                 @DataDog/agent-devx-infra
@@ -465,6 +465,7 @@
 /pkg/util/pdhutil/                      @DataDog/windows-agent
 /pkg/util/winutil/                      @DataDog/windows-agent
 /pkg/util/testutil/flake                @DataDog/agent-devx-loops
+/pkg/util/testutil/patternscanner.go    @DataDog/universal-service-monitoring @DataDog/ebpf-platform
 /pkg/util/testutil/docker               @DataDog/universal-service-monitoring @DataDog/ebpf-platform
 /pkg/util/trie                          @DataDog/container-integrations
 /pkg/languagedetection                  @DataDog/processes @DataDog/universal-service-monitoring

.github/workflows/codeql-analysis.yml

@@ -45,7 +45,7 @@ jobs:
           go-version-file: ".go-version"
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+        uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
         with:
           languages: ${{ matrix.language }}
           config: |
@@ -67,4 +67,4 @@ jobs:
           invoke agent.build --build-exclude=systemd
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
+        uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5

.gitlab-ci.yml

@@ -173,49 +173,49 @@ variables:
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
   DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v48815877-9bfad02c
+  DATADOG_AGENT_BUILDIMAGES: v50094600-7b86dd80
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v48815877-9bfad02c
+  DATADOG_AGENT_WINBUILDIMAGES: v50094600-7b86dd80
   DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v48815877-9bfad02c
+  DATADOG_AGENT_ARMBUILDIMAGES: v50094600-7b86dd80
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v48815877-9bfad02c
+  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v50094600-7b86dd80
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v48815877-9bfad02c
+  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v50094600-7b86dd80
   # New images to enable different version per image - not used yet
-  CI_IMAGE_BTF_GEN: v48815877-9bfad02c
+  CI_IMAGE_BTF_GEN: v50094600-7b86dd80
   CI_IMAGE_BTF_GEN_SUFFIX: ""
-  CI_IMAGE_DEB_X64: v48815877-9bfad02c
+  CI_IMAGE_DEB_X64: v50094600-7b86dd80
   CI_IMAGE_DEB_X64_SUFFIX: ""
-  CI_IMAGE_DEB_ARM64: v48815877-9bfad02c
+  CI_IMAGE_DEB_ARM64: v50094600-7b86dd80
   CI_IMAGE_DEB_ARM64_SUFFIX: ""
-  CI_IMAGE_DEB_ARMHF: v48815877-9bfad02c
+  CI_IMAGE_DEB_ARMHF: v50094600-7b86dd80
   CI_IMAGE_DEB_ARMHF_SUFFIX: ""
-  CI_IMAGE_DD_AGENT_TESTING: v48815877-9bfad02c
+  CI_IMAGE_DD_AGENT_TESTING: v50094600-7b86dd80
   CI_IMAGE_DD_AGENT_TESTING_SUFFIX: ""
-  CI_IMAGE_DOCKER_X64: v48815877-9bfad02c
+  CI_IMAGE_DOCKER_X64: v50094600-7b86dd80
   CI_IMAGE_DOCKER_X64_SUFFIX: ""
-  CI_IMAGE_DOCKER_ARM64: v48815877-9bfad02c
+  CI_IMAGE_DOCKER_ARM64: v50094600-7b86dd80
   CI_IMAGE_DOCKER_ARM64_SUFFIX: ""
-  CI_IMAGE_GITLAB_AGENT_DEPLOY: v48815877-9bfad02c
+  CI_IMAGE_GITLAB_AGENT_DEPLOY: v50094600-7b86dd80
   CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX: ""
-  CI_IMAGE_LINUX_GLIBC_2_17_X64: v48815877-9bfad02c
+  CI_IMAGE_LINUX_GLIBC_2_17_X64: v50094600-7b86dd80
   CI_IMAGE_LINUX_GLIBC_2_17_X64_SUFFIX: ""
-  CI_IMAGE_LINUX_GLIBC_2_23_ARM64: v48815877-9bfad02c
+  CI_IMAGE_LINUX_GLIBC_2_23_ARM64: v50094600-7b86dd80
   CI_IMAGE_LINUX_GLIBC_2_23_ARM64_SUFFIX: ""
-  CI_IMAGE_SYSTEM_PROBE_X64: v48815877-9bfad02c
+  CI_IMAGE_SYSTEM_PROBE_X64: v50094600-7b86dd80
   CI_IMAGE_SYSTEM_PROBE_X64_SUFFIX: ""
-  CI_IMAGE_SYSTEM_PROBE_ARM64: v48815877-9bfad02c
+  CI_IMAGE_SYSTEM_PROBE_ARM64: v50094600-7b86dd80
   CI_IMAGE_SYSTEM_PROBE_ARM64_SUFFIX: ""
-  CI_IMAGE_RPM_X64: v48815877-9bfad02c
+  CI_IMAGE_RPM_X64: v50094600-7b86dd80
   CI_IMAGE_RPM_X64_SUFFIX: ""
-  CI_IMAGE_RPM_ARM64: v48815877-9bfad02c
+  CI_IMAGE_RPM_ARM64: v50094600-7b86dd80
   CI_IMAGE_RPM_ARM64_SUFFIX: ""
-  CI_IMAGE_RPM_ARMHF: v48815877-9bfad02c
+  CI_IMAGE_RPM_ARMHF: v50094600-7b86dd80
   CI_IMAGE_RPM_ARMHF_SUFFIX: ""
-  CI_IMAGE_WIN_1809_X64: v48815877-9bfad02c
+  CI_IMAGE_WIN_1809_X64: v50094600-7b86dd80
   CI_IMAGE_WIN_1809_X64_SUFFIX: ""
-  CI_IMAGE_WIN_LTSC2022_X64: v48815877-9bfad02c
+  CI_IMAGE_WIN_LTSC2022_X64: v50094600-7b86dd80
   CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: ""
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 047dd64128b6
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 1f340fd450e3

.gitlab/kernel_matrix_testing/security_agent.yml

@@ -361,7 +361,7 @@ kmt_run_secagent_tests_arm64_fentry:
   parallel:
     matrix:
       - TAG:
-          - "amazon_2023"
+          - "ubuntu_24.04"
         TEST_SET: [cws_fentry]
   after_script:
     - !reference [.collect_outcomes_kmt]

.gitlab/source_test/macos.yml

@@ -40,7 +40,6 @@ tests_macos:
   rules:
     - !reference [.except_mergequeue]
     - when: on_success
-  allow_failure: true
   extends: .macos_gitlab
   needs: ["go_deps", "go_tools_deps"]
   variables:
@@ -78,6 +77,7 @@ tests_macos_gitlab_arm64:
   rules:
     !reference [.manual]
   tags: ["macos:monterey-arm64", "specific:true"]
+  allow_failure: true
   after_script:
     - !reference [.vault_login]
     - !reference [.select_python_env_commands]

LICENSE-3rdparty.csv

@@ -2143,6 +2143,7 @@ core,github.com/stormcat24/protodep/pkg/logger,Apache-2.0,Copyright Akinori Yama
 core,github.com/streadway/amqp,BSD-2-Clause,"Copyright (c) 2012-2019, Sean Treadway, SoundCloud Ltd"
 core,github.com/stretchr/objx,MIT,"Copyright (c) 2014 Stretchr, Inc | Copyright (c) 2017-2018 objx contributors"
 core,github.com/stretchr/testify/assert,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors"
+core,github.com/stretchr/testify/assert/yaml,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors"
 core,github.com/stretchr/testify/mock,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors"
 core,github.com/stretchr/testify/require,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors"
 core,github.com/stretchr/testify/suite,MIT,"Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors"

README.md

@@ -28,8 +28,7 @@ To build the Agent you need:
 **Note:** you may want to use a python virtual environment to avoid polluting your
       system-wide python environment with the agent build/dev dependencies. You can
       create a virtual environment using `virtualenv` and then use the `invoke agent.build`
-      parameters `--python-home-2=<venv_path>` and/or `--python-home-3=<venv_path>`
-      (depending on the python versions you are using) to use the virtual environment's
+      parameters `--python-home-3=<venv_path>` to use the virtual environment's
       interpreter and libraries. By default, this environment is only used for dev dependencies
       listed in `requirements.txt`.
 
@@ -55,7 +54,6 @@ To start working on the Agent, you can build the `main` branch:
      virtualenvs):
 
        invoke agent.build \
-         --python-home-2=$GOPATH/src/github.com/DataDog/datadog-agent/venv2 \
          --python-home-3=$GOPATH/src/github.com/DataDog/datadog-agent/venv3
 
     Running `invoke agent.build`:

cmd/agent/subcommands/run/command.go

@@ -89,7 +89,7 @@ import (
 	dogstatsdStatusimpl "github.com/DataDog/datadog-agent/comp/dogstatsd/status/statusimpl"
 	"github.com/DataDog/datadog-agent/comp/forwarder"
 	"github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder"
-	eventplatformfx "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/fx"
+	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl"
 	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl"
 	orchestratorForwarderImpl "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl"
 	langDetectionCl "github.com/DataDog/datadog-agent/comp/languagedetection/client"
@@ -424,7 +424,7 @@ func getSharedFxOption() fx.Option {
 		langDetectionClimpl.Module(),
 		metadata.Bundle(),
 		orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDefaultParams()),
-		eventplatformfx.Module(),
+		eventplatformimpl.Module(eventplatformimpl.NewDefaultParams()),
 		eventplatformreceiverimpl.Module(),
 
 		// injecting the shared Serializer to FX until we migrate it to a proper component. This allows other

cmd/agent/subcommands/secret/command.go

@@ -8,6 +8,7 @@ package secret
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/spf13/cobra"
 	"go.uber.org/fx"
@@ -68,14 +69,47 @@ func showSecretInfo(config config.Component, _ log.Component) error {
 }
 
 func secretRefresh(config config.Component, _ log.Component) error {
+	fmt.Println("Agent refresh:")
 	res, err := callIPCEndpoint(config, "agent/secret/refresh")
 	if err != nil {
 		return err
 	}
 	fmt.Println(string(res))
+
+	if config.GetBool("apm_config.enabled") {
+		fmt.Println("APM agent refresh:")
+		res, err = traceAgentSecretRefresh(config)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(res))
+	}
 	return nil
 }
 
+func traceAgentSecretRefresh(conf config.Component) ([]byte, error) {
+	err := apiutil.SetAuthToken(conf)
+	if err != nil {
+		return nil, err
+	}
+
+	port := conf.GetInt("apm_config.debug.port")
+	if port <= 0 {
+		return nil, fmt.Errorf("invalid apm_config.debug.port -- %d", port)
+	}
+
+	c := apiutil.GetClient(false)
+	c.Timeout = conf.GetDuration("server_timeout") * time.Second
+
+	url := fmt.Sprintf("http://127.0.0.1:%d/secret/refresh", port)
+	res, err := apiutil.DoGet(c, url, apiutil.CloseConnection)
+	if err != nil {
+		return nil, fmt.Errorf("could not contact trace-agent: %s", err)
+	}
+
+	return res, nil
+}
+
 func callIPCEndpoint(config config.Component, endpointURL string) ([]byte, error) {
 	endpoint, err := apiutil.NewIPCEndpoint(config, endpointURL)
 	if err != nil {

cmd/agent/subcommands/snmp/command.go

@@ -23,7 +23,7 @@ import (
 	nooptagger "github.com/DataDog/datadog-agent/comp/core/tagger/fx-noop"
 	"github.com/DataDog/datadog-agent/comp/forwarder"
 	"github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder"
-	eventplatformfx "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/fx"
+	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl"
 	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl"
 	"github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl"
 	haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx"
@@ -97,7 +97,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 				demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()),
 				forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithFeatures(defaultforwarder.CoreFeatures))),
 				orchestratorimpl.Module(orchestratorimpl.NewDefaultParams()),
-				eventplatformfx.Module(),
+				eventplatformimpl.Module(eventplatformimpl.NewDefaultParams()),
 				compressionfx.Module(),
 				nooptagger.Module(),
 				eventplatformreceiverimpl.Module(),
@@ -160,7 +160,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 				aggregator.Bundle(demultiplexerimpl.NewDefaultParams()),
 				orchestratorimpl.Module(orchestratorimpl.NewDefaultParams()),
 				forwarder.Bundle(defaultforwarder.NewParams(defaultforwarder.WithFeatures(defaultforwarder.CoreFeatures))),
-				eventplatformfx.Module(),
+				eventplatformimpl.Module(eventplatformimpl.NewDefaultParams()),
 				eventplatformreceiverimpl.Module(),
 				compressionfx.Module(),
 				nooptagger.Module(),

cmd/cluster-agent-cloudfoundry/subcommands/run/command.go

@@ -46,7 +46,7 @@ import (
 	workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx"
 	"github.com/DataDog/datadog-agent/comp/forwarder"
 	"github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder"
-	eventplatformfxnoop "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/fx-noop"
+	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl"
 	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl"
 	orchestratorForwarderImpl "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl"
 	haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx"
@@ -89,7 +89,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 				compressionfx.Module(),
 				demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()),
 				orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDisabledParams()),
-				eventplatformfxnoop.Module(),
+				eventplatformimpl.Module(eventplatformimpl.NewDisabledParams()),
 				eventplatformreceiverimpl.Module(),
 
 				// setup workloadmeta

cmd/cluster-agent/admission/server.go

@@ -21,9 +21,12 @@ import (
 	"github.com/cihub/seelog"
 	admiv1 "k8s.io/api/admission/v1"
 	admiv1beta1 "k8s.io/api/admission/v1beta1"
+	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
 	authenticationv1 "k8s.io/api/authentication/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 
@@ -40,14 +43,22 @@ const jsonContentType = "application/json"
 
 // Request contains the information of an admission request
 type Request struct {
-	// Raw is the raw request object
-	Raw []byte
+	// UID is the unique identifier of the AdmissionRequest
+	UID types.UID
 	// Name is the name of the object
 	Name string
 	// Namespace is the namespace of the object
 	Namespace string
+	// Kind is the kind of the object
+	Kind metav1.GroupVersionKind
+	// Operation is the operation of the request
+	Operation admissionregistrationv1.OperationType
 	// UserInfo contains information about the requesting user
 	UserInfo *authenticationv1.UserInfo
+	// Object is the new object being admitted. It is null for DELETE operations
+	Object []byte
+	// OldObject is the existing object. It is null for CREATE and CONNECT operations
+	OldObject []byte
 	// DynamicClient holds a dynamic Kubernetes client
 	DynamicClient dynamic.Interface
 	// APIClient holds a Kubernetes client
@@ -190,10 +201,14 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request, webhookName stri
 		admissionReview := &admiv1.AdmissionReview{}
 		admissionReview.SetGroupVersionKind(*gvk)
 		admissionRequest := Request{
-			Raw:           admissionReviewReq.Request.Object.Raw,
+			UID:           admissionReviewReq.Request.UID,
+			Kind:          admissionReviewReq.Request.Kind,
 			Name:          admissionReviewReq.Request.Name,
 			Namespace:     admissionReviewReq.Request.Namespace,
+			Operation:     admissionregistrationv1.OperationType(admissionReviewReq.Request.Operation),
 			UserInfo:      &admissionReviewReq.Request.UserInfo,
+			Object:        admissionReviewReq.Request.Object.Raw,
+			OldObject:     admissionReviewReq.Request.OldObject.Raw,
 			DynamicClient: dc,
 			APIClient:     apiClient,
 		}
@@ -212,10 +227,14 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request, webhookName stri
 		admissionReview := &admiv1beta1.AdmissionReview{}
 		admissionReview.SetGroupVersionKind(*gvk)
 		admissionRequest := Request{
-			Raw:           admissionReviewReq.Request.Object.Raw,
+			UID:           admissionReviewReq.Request.UID,
+			Kind:          admissionReviewReq.Request.Kind,
 			Name:          admissionReviewReq.Request.Name,
 			Namespace:     admissionReviewReq.Request.Namespace,
+			Operation:     admissionregistrationv1.OperationType(admissionReviewReq.Request.Operation),
 			UserInfo:      &admissionReviewReq.Request.UserInfo,
+			Object:        admissionReviewReq.Request.Object.Raw,
+			OldObject:     admissionReviewReq.Request.OldObject.Raw,
 			DynamicClient: dc,
 			APIClient:     apiClient,
 		}

cmd/cluster-agent/subcommands/start/command.go

@@ -58,7 +58,7 @@ import (
 	workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx"
 	"github.com/DataDog/datadog-agent/comp/forwarder"
 	"github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder"
-	eventplatformfxnoop "github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/fx-noop"
+	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatform/eventplatformimpl"
 	"github.com/DataDog/datadog-agent/comp/forwarder/eventplatformreceiver/eventplatformreceiverimpl"
 	orchestratorForwarderImpl "github.com/DataDog/datadog-agent/comp/forwarder/orchestrator/orchestratorimpl"
 	haagentfx "github.com/DataDog/datadog-agent/comp/haagent/fx"
@@ -141,7 +141,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
 				compressionfx.Module(),
 				demultiplexerimpl.Module(demultiplexerimpl.NewDefaultParams()),
 				orchestratorForwarderImpl.Module(orchestratorForwarderImpl.NewDefaultParams()),
-				eventplatformfxnoop.Module(),
+				eventplatformimpl.Module(eventplatformimpl.NewDisabledParams()),
 				eventplatformreceiverimpl.Module(),
 				// setup workloadmeta
 				wmcatalog.GetCatalog(),
@@ -479,6 +479,7 @@ func start(log log.Component,
 			Client:              apiCl.Cl,
 			StopCh:              stopCh,
 			ValidatingStopCh:    validatingStopCh,
+			Demultiplexer:       demultiplexer,
 		}
 
 		webhooks, err := admissionpkg.StartControllers(admissionCtx, wmeta, pa, datadogConfig)