Merge branch 'main' of github.com:DataDog/datadog-agent into arhodes_…

…netword_device_autodiscovery_listener_fix

.circleci/README.md

@@ -11,12 +11,12 @@ This image is now built alongside other images in [agent-buildimages](https://gi
 Once you have created a new image by building a new version of agent-buildimages, you can test your modification with the associated invoke task:
 
 ```bash
-invoke -e pipeline.update-buildimages --image-tag v12345678-c0mm1t5
+invoke -e buildimages.update --image-tag v12345678-c0mm1t5
 ```
 This will update the configuration of circleci and gitlab to use the __test version__ of these images.
 Once your test is successful, you can either move the `_test_version` from files or invoke
 ```bash
-invoke -e pipeline.update-buildimages --image-tag v12345678-c0mm1t5 --no-test-version
+invoke -e buildimages.update --image-tag v12345678-c0mm1t5 --no-test-version
 ```
 
 If everything is green, get a review and merge the PR.

.gitlab-ci.yml

@@ -30,7 +30,6 @@ include:
   - .gitlab/integration_test/include.yml
   - .gitlab/internal_image_deploy/internal_image_deploy.yml
   - .gitlab/internal_kubernetes_deploy/include.yml
-  - .gitlab/junit_upload/junit_upload.yml
   - .gitlab/kitchen_cleanup/include.yml
   - .gitlab/kitchen_deploy/kitchen_deploy.yml
   - .gitlab/kitchen_testing/include.yml
@@ -257,17 +256,19 @@ variables:
   VCPKG_BLOB_SAS_URL: ci.datadog-agent-buildimages.vcpkg_blob_sas_url  # windows-agent
   WINGET_PAT: ci.datadog-agent.winget_pat  # windows-agent
   # End aws ssm variables
- 
+
   # Start vault variables
   AGENT_API_KEY_ORG2: agent-api-key-org-2  # agent-devx-infra
   AGENT_APP_KEY_ORG2: agent-ci-app-key-org-2  # agent-devx-infra
   AGENT_GITHUB_APP: agent-github-app  # agent-devx-infra
+  AGENT_QA_E2E: agent-qa-e2e  # agent-devx-loops
   ATLASSIAN_WRITE: atlassian-write  # agent-devx-infra
   DOCKER_REGISTRY_RO: dockerhub-readonly  # agent-delivery
   GITLAB_TOKEN: gitlab-token  # agent-devx-infra
   INSTALL_SCRIPT_API_KEY_ORG2: install-script-api-key-org-2  # agent-devx-infra
   MACOS_GITHUB_APP_1: macos-github-app-one  # agent-devx-infra
   MACOS_GITHUB_APP_2: macos-github-app-two  # agent-devx-infra
+  SLACK_AGENT: slack-agent-ci  # agent-devx-infra
   # End vault variables
 
   DD_PKG_VERSION: "latest"
@@ -1188,3 +1189,12 @@ workflow:
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
   - when: manual
     allow_failure: true
+
+.on_gpu_or_e2e_changes:
+  - !reference [.on_e2e_main_release_or_rc]
+  - changes:
+      paths:
+        - pkg/gpu/**/*
+        - test/new-e2e/tests/gpu/**/*
+        - pkg/collector/corechecks/gpu/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916

.gitlab/.ci-linters.yml

@@ -109,7 +109,6 @@ job-owners:
     - trace-agent-v05-2cpus-normal_load-fixed_sps
     - trace-agent-v05-2cpus-stress_load-fixed_sps
     - trigger-flakes-finder
-    - unit_tests_arm64_junit_upload
     - unit_tests_notify
     - update_rc_build_links
     - validate_modules

.gitlab/JOBOWNERS

@@ -147,30 +147,9 @@ k8s-e2e-otlp-*                    @DataDog/opentelemetry
 k8s-e2e-cspm-*                    @DataDog/agent-security
 
 # New E2E
-new-e2e-containers*                    @DataDog/container-integrations
-new-e2e-agent-subcommands*             @DataDog/agent-shared-components
-new-e2e-agent-shared-components*       @DataDog/agent-shared-components
-new-e2e-language-detection*            @DataDog/processes
-new-e2e-process*                       @DataDog/processes
-new-e2e-agent-platform*                @DataDog/container-ecosystems @DataDog/agent-delivery
-new-e2e-platform-integrations*         @DataDog/agent-delivery @DataDog/platform-integrations
-new-e2e-aml*                           @DataDog/agent-metrics-logs
-new-e2e-apm*                           @DataDog/agent-apm
-new-e2e-discovery*                     @Datadog/universal-service-monitoring
-new-e2e-ndm*                           @DataDog/network-device-monitoring
-new-e2e-npm*                           @DataDog/Networks
-new-e2e-cws*                           @DataDog/agent-security
-new-e2e-orchestrator*                  @DataDog/container-app
-new-e2e-otel*                          @DataDog/opentelemetry
-e2e_pre_test*                          @DataDog/agent-devx-loops
-new-e2e-remote-config*                 @DataDog/remote-config
-new-e2e-installer*                     @DataDog/fleet
-new-e2e-installer-windows              @DataDog/windows-agent
-new-e2e-windows*                       @DataDog/windows-agent
-new-e2e-windows-systemprobe            @DataDog/windows-kernel-integrations
-new-e2e-windows-security-agent         @DataDog/windows-kernel-integrations
-new-e2e_windows_powershell_module_test @DataDog/windows-kernel-integrations
-new-e2e-eks-cleanup-on-failure         @DataDog/agent-devx-loops
+e2e_pre_test*                     @DataDog/agent-devx-loops
+new-e2e*                          @DataDog/multiple
+
 
 # Kernel matrix testing
 upload_dependencies*              @DataDog/ebpf-platform

.gitlab/choco_deploy/choco_deploy.yml

@@ -12,7 +12,7 @@ publish_choco_7_x64:
   before_script:
     - $tmpfile = [System.IO.Path]::GetTempFileName()
     - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:CHOCOLATEY_API_KEY" -tempFile "$tmpfile")
-    - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
+    - If ($lastExitCode -ne "0") { exit "$lastExitCode" }
     - $chocolateyApiKey=$(cat "$tmpfile")
     - Remove-Item "$tmpfile"
   script:

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: f694c4dc33e4
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: cd9a362371a8

.gitlab/container_build/docker_windows.yml

@@ -34,8 +34,8 @@
       -v "$(Get-Location):C:\mnt"
       -v \\.\pipe\docker_engine:\\.\pipe\docker_engine 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_${Env:VARIANT}_x64${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
       powershell
-      -C C:\mnt\tools\ci\docker-login.ps1
-    - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
+      -File C:\mnt\tools\ci\docker-login.ps1
+    - If ($lastExitCode -ne "0") { exit "$lastExitCode" }
     - powershell -Command "$(Get-Location)\tools\ci\retry.ps1 docker build --no-cache --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} ${BUILD_ARG} --pull --file ${BUILD_CONTEXT}/windows/amd64/Dockerfile --tag ${TARGET_TAG} ${BUILD_CONTEXT}"
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
     - powershell -Command "$(Get-Location)\tools\ci\retry.ps1 docker push ${TARGET_TAG}"

.gitlab/deploy_packages/winget.yml

@@ -12,7 +12,7 @@ publish_winget_7_x64:
   before_script:
     - $tmpfile = [System.IO.Path]::GetTempFileName()
     - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:WINGET_PAT" -tempFile "$tmpfile")
-    - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
+    - If ($lastExitCode -ne "0") { exit "$lastExitCode" }
     - $wingetPat=$(cat "$tmpfile")
     - Remove-Item "$tmpfile"
   script:

.gitlab/e2e/e2e.yml

@@ -11,11 +11,11 @@
     - !reference [.retrieve_linux_go_e2e_deps]
     # Setup AWS Credentials
     - mkdir -p ~/.aws
-    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $?
+    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $?
     - export AWS_PROFILE=agent-qa-ci
     # Now all `aws` commands target the agent-qa profile
-    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_PUBLIC_KEY_RSA > $E2E_PUBLIC_KEY_PATH || exit $?
-    - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_KEY_RSA > $E2E_PRIVATE_KEY_PATH || exit $?
+    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_PUBLIC_KEY_PATH || exit $?
+    - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_PRIVATE_KEY_PATH || exit $?
     # Use S3 backend
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
     # Setup Azure credentials. https://www.pulumi.com/registry/packages/azure-native/installation-configuration/#set-configuration-using-pulumi-config
@@ -120,6 +120,7 @@ new-e2e-containers-eks-init:
     TEAM: container-integrations
     EXTRA_PARAMS: --run TestEKSSuite
     E2E_INIT_ONLY: "true"
+    SHOULD_RUN_IN_FLAKES_FINDER: "false"
   allow_failure: true
 
 new-e2e-containers-eks:
@@ -256,6 +257,7 @@ new-e2e-npm-eks-init:
     TEAM: network-performance-monitoring
     EXTRA_PARAMS: --run "TestEKSVMSuite"
     E2E_INIT_ONLY: "true"
+    SHOULD_RUN_IN_FLAKES_FINDER: "false"
   allow_failure: true
 
 new-e2e-npm-eks:
@@ -302,6 +304,7 @@ new-e2e-cws:
     - qa_agent
     - qa_dca
   variables:
+    SHOULD_RUN_IN_FLAKES_FINDER: "false" # Currently broken in flake finder ADXT-687
     TARGETS: ./tests/cws
     TEAM: csm-threats-agent
     CWS_INSTRUMENTATION_FULLIMAGEPATH: 669783387624.dkr.ecr.us-east-1.amazonaws.com/cws-instrumentation:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
@@ -471,6 +474,7 @@ new-e2e-windows-systemprobe:
   variables:
     TARGETS: ./tests/sysprobe-functional
     TEAM: windows-kernel-integrations
+    SHOULD_RUN_IN_FLAKES_FINDER: "false" # Currently broken in flake finder ADXT-687
 
 new-e2e-windows-security-agent:
   extends: .new_e2e_template
@@ -493,14 +497,13 @@ new-e2e-otel-eks-init:
     - !reference [.manual]
   needs:
     - !reference [.needs_new_e2e_template]
-    - qa_dca
-    - qa_agent
-    - qa_agent_ot
   variables:
     TARGETS: ./tests/otel
     TEAM: otel
     EXTRA_PARAMS: --run TestOTelAgentIAEKS
     E2E_INIT_ONLY: "true"
+    SHOULD_RUN_IN_FLAKES_FINDER: "false"
+
   allow_failure: true
 
 new-e2e-otel-eks:
@@ -569,6 +572,19 @@ new-e2e-cspm:
     TEAM: cspm
   timeout: 35m
 
+new-e2e-gpu:
+  extends: .new_e2e_template
+  rules:
+    - !reference [.on_gpu_or_e2e_changes]
+    - !reference [.manual]
+  variables:
+    TARGETS: ./tests/gpu                          # the target path where tests are
+    TEAM: ebpf-platform
+  needs:                                         # list of required jobs. By default gitlab waits for any previous jobs.
+    - !reference [.needs_new_e2e_template]
+    - deploy_deb_testing-a7_x64                  # agent 7 debian package
+
+
 generate-flakes-finder-pipeline:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   stage: e2e
@@ -623,11 +639,11 @@ new-e2e-eks-cleanup-on-failure:
   script:
     # Setup AWS Credentials
     - mkdir -p ~/.aws
-    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $?
+    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $?
     - export AWS_PROFILE=agent-qa-ci
     # Now all `aws` commands target the agent-qa profile
-    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_PUBLIC_KEY_RSA > $E2E_PUBLIC_KEY_PATH || exit $?
-    - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_KEY_RSA > $E2E_PRIVATE_KEY_PATH || exit $?
+    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_PUBLIC_KEY_PATH || exit $?
+    - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_PRIVATE_KEY_PATH || exit $?
     # Use S3 backend
     - PULUMI_CONFIG_PASSPHRASE=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_PULUMI_CONFIG_PASSPHRASE) || exit $?; export PULUMI_CONFIG_PASSPHRASE
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"

.gitlab/integration_test/windows.yml

@@ -9,7 +9,7 @@
   before_script:
     - $tmpfile = [System.IO.Path]::GetTempFileName()
     - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:VCPKG_BLOB_SAS_URL" -tempFile "$tmpfile")
-    - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
+    - If ($lastExitCode -ne "0") { exit "$lastExitCode" }
     - $vcpkgBlobSaSUrl=$(cat "$tmpfile")
     - Remove-Item "$tmpfile"
   script:

.gitlab/kernel_matrix_testing/common.yml

@@ -29,7 +29,7 @@
 
 .write_ssh_key_file:
   - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE
-  - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_KEY > $AWS_EC2_SSH_KEY_FILE || exit $?
+  - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE || exit $?
   # Without the newline ssh silently fails and moves on to try other auth methods
   - echo "" >> $AWS_EC2_SSH_KEY_FILE
   - chmod 600 $AWS_EC2_SSH_KEY_FILE
@@ -47,7 +47,7 @@
 
 .kmt_new_profile:
   - mkdir -p ~/.aws
-  - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $?
+  - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $?
   - export AWS_PROFILE=agent-qa-ci
 
 .define_if_collect_complexity:

.gitlab/kernel_matrix_testing/security_agent.yml

@@ -32,7 +32,7 @@ kmt_setup_env_secagent_arm64:
     AMI_ID_ARG: "--arm-ami-id=$KERNEL_MATRIX_TESTING_ARM_AMI_ID"
     LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-arm
     TEST_COMPONENT: security-agent
-    TEST_SETS: cws_host
+    TEST_SETS: cws_host,cws_docker
 
 kmt_setup_env_secagent_x64:
   extends:
@@ -167,7 +167,25 @@ kmt_run_secagent_tests_x64_docker:
   parallel:
     matrix:
       - TAG:
+          - "ubuntu_18.04"
+          - "ubuntu_20.04"
+          - "ubuntu_22.04"
+          - "ubuntu_23.10"
           - "ubuntu_24.04"
+          - "amazon_4.14"
+          - "amazon_5.4"
+          - "amazon_5.10"
+          - "amazon_2023"
+          - "fedora_37"
+          - "fedora_38"
+          - "debian_10"
+          - "debian_11"
+          - "debian_12"
+          - "centos_7.9"
+          - "oracle_8.9"
+          - "oracle_9.3"
+          - "rocky_8.5"
+          - "rocky_9.3"
         TEST_SET: [cws_docker]
   after_script:
     - !reference [.collect_outcomes_kmt]
@@ -207,6 +225,39 @@ kmt_run_secagent_tests_arm64:
     - !reference [.collect_outcomes_kmt]
     - !reference [.upload_junit_kmt]
 
+kmt_run_secagent_tests_arm64_docker:
+  extends:
+    - .kmt_run_secagent_tests
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
+  tags: ["arch:arm64"]
+  needs:
+    - kmt_setup_env_secagent_arm64
+    - upload_dependencies_secagent_arm64
+    - upload_secagent_tests_arm64
+  variables:
+    ARCH: "arm64"
+  parallel:
+    matrix:
+      - TAG:
+          - "ubuntu_22.04"
+          - "ubuntu_23.10"
+          - "ubuntu_24.04"
+          - "amazon_5.4"
+          - "amazon_5.10"
+          - "amazon_2023"
+          - "fedora_37"
+          - "fedora_38"
+          - "debian_11"
+          - "debian_12"
+          - "oracle_8.9"
+          - "oracle_9.3"
+          - "rocky_8.5"
+          - "rocky_9.3"
+        TEST_SET: ["cws_docker"]
+  after_script:
+    - !reference [.collect_outcomes_kmt]
+    - !reference [.upload_junit_kmt]
+
 .kmt_secagent_cleanup:
   extends:
     - .kmt_cleanup

.gitlab/notify/notify.yml

@@ -114,7 +114,7 @@ notify_gitlab_ci_changes:
   timeout: 15 minutes # Added to prevent a stuck job blocking the resource_group defined above
 
 .failure_summary_setup:
-  - SLACK_API_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT_CI_TOKEN) || exit $?; export SLACK_API_TOKEN
+  - SLACK_API_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT token) || exit $?; export SLACK_API_TOKEN
   - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN
   - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY
   - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt

.gitlab/source_test/linux.yml

@@ -124,6 +124,7 @@ tests_deb-arm64-py3:
     - .rtloader_tests
     - .linux_tests
   after_script:
+    - !reference [.upload_junit_source]
     - !reference [.upload_coverage]
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:arm64"]
@@ -136,6 +137,7 @@ tests_rpm-arm64-py3:
     - .rtloader_tests
     - .linux_tests
   after_script:
+    - !reference [.upload_junit_source]
     - !reference [.upload_coverage]
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:arm64"]
@@ -172,7 +174,7 @@ new-e2e-unit-tests:
     - !reference [.retrieve_linux_go_e2e_deps]
     # Setup AWS Credentials
     - mkdir -p ~/.aws
-    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $?
+    - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $?
     - export AWS_PROFILE=agent-qa-ci
     # Use S3 backend
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"

.gitlab/source_test/windows.yml

@@ -46,7 +46,7 @@
       -e COVERAGE_CACHE_FLAG="${COVERAGE_CACHE_FLAG}"
       486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
       c:\mnt\tasks\winbuildscripts\unittests.bat
-    - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
+    - If ($lastExitCode -ne "0") { exit "$lastExitCode" }
   variables:
     TEST_OUTPUT_FILE: test_output.json
   artifacts: