Merge branch 'main' into JakeYankovich/startDDservices-exception-log

.custom-gcl.yml

@@ -0,0 +1,7 @@
+version: v1.59.1
+
+name: golangci-lint
+
+plugins:
+  - module: 'github.com/DataDog/datadog-agent/pkg/linters/components/pkgconfigusage'
+    path: ./pkg/linters/components/pkgconfigusage

.github/CODEOWNERS

@@ -19,7 +19,9 @@
 /.go-version                            @DataDog/agent-shared-components @DataDog/agent-delivery
 # Go linters and pre-commit config
 /.golangci.yml                          @DataDog/agent-devx-loops
+/.custom-gcl.yml                        @DataDog/agent-devx-loops
 /.pre-commit-config.yaml                @DataDog/agent-devx-loops
+/.vscode/                               @DataDog/agent-devx-loops
 
 /CHANGELOG.rst                          @DataDog/agent-delivery
 /CHANGELOG-DCA.rst                      @DataDog/container-integrations @DataDog/container-platform
@@ -196,6 +198,7 @@
 /cmd/system-probe/modules/traceroute*   @DataDog/network-device-monitoring @Datadog/Networks
 /cmd/system-probe/modules/ping*         @DataDog/network-device-monitoring
 /cmd/system-probe/modules/service_discover* @DataDog/apm-onboarding @DataDog/universal-service-monitoring
+/cmd/system-probe/modules/language_detection* @DataDog/processes @DataDog/universal-service-monitoring
 /cmd/system-probe/runtime/              @DataDog/agent-security
 /cmd/system-probe/windows/              @DataDog/windows-kernel-integrations
 /cmd/system-probe/windows_resources/    @DataDog/windows-kernel-integrations
@@ -293,12 +296,15 @@
 /pkg/cli/                               @DataDog/agent-shared-components
 /pkg/cli/subcommands/clusterchecks      @DataDog/container-platform
 /pkg/dogstatsd/                         @DataDog/agent-metrics-logs
+/pkg/discovery/				@DataDog/apm-onboarding @DataDog/universal-service-monitoring
 /pkg/errors/                            @DataDog/agent-shared-components
 /pkg/forwarder/                         @DataDog/agent-processing-and-routing
 /pkg/gohai                              @DataDog/agent-shared-components
 /pkg/jmxfetch/                          @DataDog/agent-metrics-logs
 /pkg/metrics/                           @DataDog/agent-metrics-logs
+/pkg/metrics/metricsource.go            @DataDog/agent-metrics-logs @DataDog/agent-integrations
 /pkg/serializer/                        @DataDog/agent-processing-and-routing
+/pkg/serializer/internal/metrics/origin_mapping.go  @DataDog/agent-processing-and-routing @DataDog/agent-integrations
 /pkg/serverless/                        @DataDog/serverless
 /pkg/serverless/appsec/                 @DataDog/asm-go
 /pkg/status/                            @DataDog/agent-shared-components
@@ -356,7 +362,7 @@
 /pkg/collector/corechecks/system/winkmem/         @DataDog/windows-agent
 /pkg/collector/corechecks/system/winproc/         @DataDog/windows-agent
 /pkg/collector/corechecks/systemd/      @DataDog/agent-integrations
-/pkg/collector/corechecks/nvidia/       @DataDog/agent-devx-infra
+/pkg/collector/corechecks/nvidia/       @DataDog/platform-integrations
 /pkg/collector/corechecks/windows_event_log/ @DataDog/windows-agent
 /pkg/config/                            @DataDog/agent-shared-components
 /pkg/config/config_template.yaml        @DataDog/agent-shared-components @DataDog/documentation
@@ -398,7 +404,6 @@
 /pkg/proto/datadog/workloadmeta         @DataDog/container-platform
 /pkg/remoteconfig/                      @DataDog/remote-config
 /pkg/runtime/                           @DataDog/agent-shared-components
-/pkg/serializer/                        @DataDog/agent-processing-and-routing
 /pkg/tagset/                            @DataDog/agent-shared-components
 /pkg/util/                              @DataDog/agent-shared-components
 /pkg/util/aggregatingqueue              @DataDog/container-integrations @DataDog/container-platform
@@ -426,6 +431,8 @@
 /pkg/util/testutil/flake                @DataDog/agent-devx-loops
 /pkg/util/trie                          @DataDog/container-integrations
 /pkg/languagedetection                  @DataDog/processes @DataDog/universal-service-monitoring
+/pkg/linters/                           @DataDog/agent-devx-loops
+/pkg/linters/components/                @DataDog/agent-shared-components
 /pkg/logs/                              @DataDog/agent-metrics-logs
 /pkg/logs/launchers/windowsevent        @DataDog/agent-metrics-logs @DataDog/windows-agent
 /pkg/logs/tailers/windowsevent          @DataDog/agent-metrics-logs @DataDog/windows-agent
@@ -587,6 +594,7 @@
 /test/new-e2e/tests/agent-shared-components   @DataDog/agent-shared-components
 /test/new-e2e/tests/agent-subcommands         @DataDog/agent-shared-components
 /test/new-e2e/tests/containers                @DataDog/container-integrations @DataDog/container-platform
+/test/new-e2e/tests/discovery                 @DataDog/apm-onboarding @DataDog/universal-service-monitoring
 /test/new-e2e/tests/language-detection        @DataDog/processes
 /test/new-e2e/tests/ndm                       @DataDog/network-device-monitoring
 /test/new-e2e/tests/npm                       @DataDog/Networks

.github/workflows/buildimages-update.yml

@@ -58,7 +58,7 @@ jobs:
           python-version: 3.11
           cache: "pip"
 
-      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           # use the go version from the input, not from the .go-version file
           # in case it's a Go update PR

.github/workflows/codeql-analysis.yml

@@ -32,20 +32,23 @@ jobs:
 
       - name: Setup env variables
         run: |
-          echo "CODEQL_PYTHON=$(which python3)" >> $GITHUB_ENV
-          echo "$GOPATH/bin" >> $GITHUB_PATH
           echo "CGO_LDFLAGS= -L${GITHUB_WORKSPACE}/rtloader/build/rtloader -ldl " >> $GITHUB_ENV
           echo "CGO_CFLAGS= -I${GITHUB_WORKSPACE}/rtloader/include  -I${GITHUB_WORKSPACE}/rtloader/common " >> $GITHUB_ENV
 
-      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version-file: ".go-version"
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15
         with:
           languages: ${{ matrix.language }}
-          setup-python-dependencies: false
+          config: |
+            paths-ignore:
+              - rtloader/build/rtloader/CMakeFiles/datadog-agent-rtloader.dir
+              - rtloader/build/three/CMakeFiles/datadog-agent-three.dir
+              - rtloader/build/test/CMakeFiles/run.dir
+              - rtloader/build/CMakeFiles/clang-format.dir
 
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c

.github/workflows/create_rc_pr.yml

@@ -6,14 +6,14 @@ on:
     - cron: '0 14 * * 1,3,5' # Run on Monday, Wednesday, and Friday at 14:00 UTC
     - cron: '0 8 * * 1,3,5' # Same as above but at 08:00 UTC, to warn agent-integrations team about releasing
 
-
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
-    create_rc_pr:
+    find_release_branches:
         runs-on: ubuntu-latest
-
+        outputs:
+          branches: ${{ steps.branches.outputs.value }}
         steps:
             - name: Checkout repository
               uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -33,27 +33,48 @@ jobs:
                 pip install -r tasks/libs/requirements-github.txt
                 pip install -r tasks/requirements_release_tasks.txt
 
-            - name: Determine the release active branch
+            - name: Determine the release active branches
+              id: branches
               run: |
-                echo "RELEASE_BRANCH=$(inv -e release.get-active-release-branch)" >> $GITHUB_ENV
+                echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT
 
             - name: Set the warning option
               if: github.event.schedule == '0 8 * * 1,3,5'
               run: echo "WARNING='-w'" >> $GITHUB_ENV
 
+    create_rc_pr:
+      runs-on: ubuntu-latest
+      needs: find_release_branches
+      strategy:
+        matrix:
+          value: ${{fromJSON(needs.find_release_branches.outputs.branches)}}
+      steps:
             - name: Checkout release branch
               uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
               with:
-                ref: ${{ env.RELEASE_BRANCH }}
+                ref: ${{ matrix.value }}
                 fetch-depth: 0
 
+            - name: Install python
+              uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+              with:
+                python-version: 3.11
+                cache: "pip"
+
+            - name: Install Python dependencies
+              run: |
+                python -m pip install --upgrade pip
+                pip install -r requirements.txt
+                pip install -r tasks/libs/requirements-github.txt
+                pip install -r tasks/requirements_release_tasks.txt
+
             - name: Check for changes since last RC
               id: check_for_changes
               env:
                 ATLASSIAN_USERNAME: ${{ secrets.ATLASSIAN_USERNAME }}
                 ATLASSIAN_PASSWORD: ${{ secrets.ATLASSIAN_PASSWORD }}
               run: |
-                echo "CHANGES=$(inv -e release.check-for-changes -r ${{ env.RELEASE_BRANCH }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT
+                echo "CHANGES=$(inv -e release.check-for-changes -r ${{ matrix.value }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT
 
             - name: Create RC PR
               if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}}

.github/workflows/cws-btfhub-sync.yml

@@ -69,7 +69,7 @@ jobs:
       - run: pip install -r requirements.txt
 
       - name: Install go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version-file: '.go-version'
 
@@ -83,7 +83,7 @@ jobs:
           inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./${{ steps.artifact-name.outputs.ARTIFACT_NAME }}.json ${{ inputs.force_refresh && '--force-refresh' || '' }}
 
       - name: Upload artifact
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
         with:
           name: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
           path: ./${{ steps.artifact-name.outputs.ARTIFACT_NAME }}.json
@@ -105,7 +105,7 @@ jobs:
       - run: pip install -r requirements.txt
 
       - name: Install go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version-file: '.go-version'
 

.github/workflows/docs-dev.yml

@@ -47,7 +47,7 @@ jobs:
     - name: Build documentation
       run: invoke docs.build
 
-    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+    - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
       with:
         name: documentation
         path: site

.github/workflows/go_mod_tidy.yml

@@ -26,7 +26,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Install go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version-file: ".go-version"
       - name: Install python

.github/workflows/gohai.yml

@@ -31,7 +31,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
-      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version-file: ${{ matrix.go-file }}
       - name: Test

.github/workflows/serverless-benchmarks.yml

@@ -27,7 +27,7 @@ jobs:
           ref: ${{ github.base_ref }}
 
       - name: Install Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version: stable
 
@@ -43,7 +43,7 @@ jobs:
               ./pkg/serverless/... | tee ${{runner.temp}}/benchmark.log
 
       - name: Upload result artifact
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
         with:
           name: baseline.log
           path: ${{runner.temp}}/benchmark.log
@@ -63,7 +63,7 @@ jobs:
           ref: ${{ github.sha }}
 
       - name: Install Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version: stable
 
@@ -79,7 +79,7 @@ jobs:
               ./pkg/serverless/... | tee ${{runner.temp}}/benchmark.log
 
       - name: Upload result artifact
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
         with:
           name: current.log
           path: ${{runner.temp}}/benchmark.log
@@ -89,10 +89,12 @@ jobs:
     name: Summary
     runs-on: ubuntu-latest
     needs: [baseline, current]
+    permissions:
+      pull-requests: write
 
     steps:
       - name: Install Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           go-version: stable
           cache: false

.github/workflows/serverless-binary-size.yml

@@ -82,14 +82,19 @@ jobs:
         id: should
         run: |
           cd go/src/github.com/DataDog/datadog-agent
+          git fetch origin $GITHUB_BASE_REF
+          git fetch origin $GITHUB_HEAD_REF
           if test $(
-            git diff $GITHUB_BASE_REF..$GITHUB_SHA --name-only | grep dependencies_linux_amd64.txt
+            git diff origin/$GITHUB_BASE_REF...origin/$GITHUB_HEAD_REF --name-only | grep dependencies_linux_amd64.txt
           ); then
             echo "should_run=true" >> $GITHUB_OUTPUT
+            echo "dependencies list changed"
           elif [[ ${{ steps.compare.outputs.diff }} > env.SIZE_ALLOWANCE ]]; then
             echo "should_run=true" >> $GITHUB_OUTPUT
+            echo "binary size changed"
           else
             echo "should_run=false" >> $GITHUB_OUTPUT
+            echo "nothing changed"
           fi
 
       ### Steps below run if size diff > SIZE_ALLOWANCE or file dependencies_linux_amd64.txt changed ###
@@ -127,7 +132,7 @@ jobs:
           done
 
       - name: Archive dependency graphs
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
         if: steps.should.outputs.should_run == 'true'
         with:
           name: dependency-graphs

.github/workflows/serverless-integration.yml

@@ -74,7 +74,7 @@ jobs:
 
       - name: Archive raw logs
         if: always()
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
         with:
           name: rawlogs-${{ matrix.suite }}-${{ matrix.architecture }}
           path: ${{ steps.rawlogs.outputs.dir }}

.gitlab-ci.yml

@@ -206,6 +206,10 @@ variables:
   E2E_TESTS_API_KEY_SSM_NAME: ci.datadog-agent.e2e_tests_api_key  # agent-devx-loops
   E2E_TESTS_APP_KEY_SSM_NAME: ci.datadog-agent.e2e_tests_app_key  # agent-devx-loops
   E2E_TESTS_RC_KEY_SSM_NAME: ci.datadog-agent.e2e_tests_rc_key  # agent-devx-loops
+  E2E_TESTS_AZURE_CLIENT_ID: ci.datadog-agent.e2e_tests_azure_client_id  # agent-devx-loops
+  E2E_TESTS_AZURE_CLIENT_SECRET: ci.datadog-agent.e2e_tests_azure_client_secret  # agent-devx-loops
+  E2E_TESTS_AZURE_TENANT_ID: ci.datadog-agent.e2e_tests_azure_tenant_id  # agent-devx-loops
+  E2E_TESTS_AZURE_SUBSCRIPTION_ID: ci.datadog-agent.e2e_tests_azure_subscription_id  # agent-devx-loops
   KITCHEN_EC2_SSH_KEY_SSM_NAME: ci.datadog-agent.aws_ec2_kitchen_ssh_key  # agent-devx-loops
   KITCHEN_AZURE_CLIENT_ID_SSM_NAME: ci.datadog-agent.azure_kitchen_client_id  # agent-devx-loops
   KITCHEN_AZURE_CLIENT_SECRET_SSM_NAME: ci.datadog-agent.azure_kitchen_client_secret  # agent-devx-loops
@@ -384,6 +388,7 @@ workflow:
         GO_TEST_SKIP_FLAKE: "false"
     - <<: *if_release_branch
     - <<: *if_deploy
+    - <<: *if_deploy_installer
     - if: $CI_COMMIT_TAG == null
 
 #
@@ -590,7 +595,6 @@ workflow:
 .except_deploy:
   - <<: *if_deploy
     when: never
-  - when: on_success
 
 .except_no_tests_no_deploy:
   - if: $DEPLOY_AGENT == "false" && $DDR_WORKFLOW_ID == null && $RUN_E2E_TESTS == "off"
@@ -720,16 +724,6 @@ workflow:
         - tasks/msi.py
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
-.on_windows_installer_changes_or_manual:
-  - <<: *if_disable_e2e_tests
-    when: never
-  - <<: *if_main_branch
-  - !reference [.except_mergequeue]
-  - <<: *if_windows_installer_changes
-    when: on_success
-  - when: manual
-    allow_failure: true
-
 .except_windows_installer_changes:
   - <<: *if_windows_installer_changes
     when: never
@@ -796,6 +790,11 @@ workflow:
         - test/new-e2e/go.mod
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
+.on_e2e_or_windows_installer_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - <<: *if_windows_installer_changes
+    when: on_success
+
 .on_e2e_or_fakeintake_changes_or_manual:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:
@@ -884,6 +883,14 @@ workflow:
         - test/new-e2e/tests/npm/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
+.on_discovery_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - test/new-e2e/tests/discovery/**/*
+        - pkg/collector/corechecks/servicediscovery/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+
 .on_aml_or_e2e_changes:
   - !reference [.on_e2e_main_release_or_rc]
   - changes: