Merge branch 'main' of github.com:DataDog/datadog-agent into pgimalac…

…/remove-seelog-outside-pkg-util-log

.github/CODEOWNERS

@@ -75,6 +75,7 @@
 /.gitlab/deploy*/*                                   @DataDog/agent-delivery
 /.gitlab/deps_fetch/*                                @DataDog/agent-devx-infra
 /.gitlab/e2e/*                                       @DataDog/agent-devx-infra @DataDog/agent-devx-loops
+/.gitlab/e2e_testing_deploy/*                        @DataDog/agent-devx-loops @DataDog/agent-devx-infra
 /.gitlab/e2e_install_packages/*                      @DataDog/agent-delivery
 /.gitlab/e2e_pre_test/*                              @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/kernel_matrix_testing/*                     @DataDog/agent-devx-infra @DataDog/ebpf-platform
@@ -90,7 +91,6 @@
 /.gitlab/binary_build/include.yml                    @DataDog/agent-devx-infra
 /.gitlab/binary_build/linux.yml                      @DataDog/agent-devx-infra @DataDog/agent-delivery
 /.gitlab/functional_test/include.yml                 @DataDog/agent-devx-infra
-/.gitlab/functional_test_cleanup/functional_test_cleanup.yml        @DataDog/agent-devx-infra @DataDog/agent-security
 /.gitlab/install_script_testing/install_script_testing.yml          @DataDog/agent-delivery
 /.gitlab/integration_test/dogstatsd.yml              @DataDog/agent-devx-infra @DataDog/agent-metrics-logs
 /.gitlab/integration_test/include.yml                @DataDog/agent-devx-infra
@@ -131,8 +131,6 @@
 
 /.gitlab/deps_build/                                 @DataDog/ebpf-platform @DataDog/agent-delivery @DataDog/windows-agent
 
-/.gitlab/kitchen_*                                   @DataDog/container-ecosystems @DataDog/agent-delivery
-/.gitlab/kitchen_testing/windows.yml                 @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
 /.gitlab/e2e_install_packages/windows.yml            @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
 
 /.gitlab/common/                                     @DataDog/agent-devx-infra
@@ -144,7 +142,6 @@
 /.gitlab/binary_build/fakeintake.yml                 @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 
 /.gitlab/functional_test/serverless.yml              @DataDog/serverless @Datadog/serverless-aws @DataDog/agent-devx-infra
-/.gitlab/functional_test_cleanup                     @DataDog/agent-security @DataDog/windows-kernel-integrations @DataDog/agent-devx-infra
 /.gitlab/functional_test/oracle.yml                  @DataDog/agent-devx-infra @DataDog/database-monitoring
 
 /.gitlab/powershell_script_deploy                    @DataDog/agent-delivery @DataDog/windows-agent
@@ -243,6 +240,7 @@
 /cmd/systray/                           @DataDog/windows-agent
 /cmd/security-agent/                    @DataDog/agent-security
 /cmd/installer/                         @DataDog/fleet @DataDog/windows-agent
+/cmd/installer-downloader/              @DataDog/fleet
 
 /dev/                                   @DataDog/agent-devx-loops
 /devenv/                                @DataDog/agent-devx-loops
@@ -454,6 +452,7 @@
 /pkg/util/docker/                       @DataDog/container-integrations
 /pkg/util/ecs/                          @DataDog/container-integrations
 /pkg/util/funcs/                        @DataDog/ebpf-platform
+/pkg/util/gpu/                          @DataDog/container-platform
 /pkg/util/kernel/                       @DataDog/ebpf-platform
 /pkg/util/safeelf/                      @DataDog/ebpf-platform
 /pkg/util/ktime                         @DataDog/agent-security
@@ -589,18 +588,6 @@
 /test/integration/docker/otel_agent_build_tests.py @DataDog/opentelemetry
 /test/integration/serverless            @DataDog/serverless @Datadog/serverless-aws
 /test/integration/serverless_perf       @DataDog/serverless @Datadog/serverless-aws
-/test/kitchen/                          @DataDog/agent-devx-loops
-/test/kitchen/test-definitions/         @DataDog/container-ecosystems @DataDog/agent-delivery
-/test/kitchen/test/integration/         @DataDog/container-ecosystems @DataDog/agent-delivery
-/test/kitchen/site-cookbooks/dd-system-probe-check/ @DataDog/windows-kernel-integrations
-/test/kitchen/test/integration/win-all-subservices/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-alt-dir/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-install-fail/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-installopts/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-no-subservices/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-sysprobe-test/ @DataDog/windows-kernel-integrations
-/test/kitchen/test/integration/win-repair/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-user/ @DataDog/windows-agent
 /test/fakeintake/                             @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/ndm-integrations
 /test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/ndm-integrations

.github/workflows/create_rc_pr.yml

@@ -96,8 +96,10 @@ jobs:
 
             - name: Create RC PR
               if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}}
+              env:
+                MATRIX: ${{ matrix.value }}
               run: |
                 git config user.name "github-actions[bot]"
                 git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
                 git fetch
-                inv -e release.create-rc --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }}
+                inv -e release.create-rc "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }}

.github/workflows/gohai.yml

@@ -7,9 +7,11 @@ on:
       - main
       - "[0-9]+.[0-9]+.x"
     paths:
+      - ".github/workflows/gohai.yml"
       - "pkg/gohai/**"
   pull_request:
     paths:
+      - ".github/workflows/gohai.yml"
       - "pkg/gohai/**"
 
 permissions: {}
@@ -23,7 +25,7 @@ jobs:
           [
             ubuntu-20.04,
             ubuntu-latest,
-            macos-12,
+            macos-13,
             macos-latest,
             windows-2019,
             windows-latest,

.github/workflows/serverless-integration.yml

@@ -9,6 +9,7 @@ on:
       - 'pkg/serverless/**'
       - 'test/integration/serverless/**'
       - '.github/workflows/serverless-integration.yml'
+      - 'go.mod'
   schedule:
     - cron: '0 14 * * *' # cron schedule uses UTC timezone. Run tests at the beginning of the day in US-East
 
@@ -67,6 +68,7 @@ jobs:
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.SERVERLESS_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.SERVERLESS_AWS_SECRET_ACCESS_KEY }}
+          GOWORK: off
         with:
           timeout_minutes: 60
           max_attempts: 2

.gitlab-ci.yml

@@ -24,15 +24,12 @@ include:
   - .gitlab/e2e/e2e.yml
   - .gitlab/e2e_install_packages/include.yml
   - .gitlab/e2e_pre_test/e2e_pre_test.yml
+  - .gitlab/e2e_testing_deploy/e2e_deploy.yml
   - .gitlab/functional_test/include.yml
-  - .gitlab/functional_test_cleanup/functional_test_cleanup.yml
   - .gitlab/install_script_testing/install_script_testing.yml
   - .gitlab/integration_test/include.yml
   - .gitlab/internal_image_deploy/internal_image_deploy.yml
   - .gitlab/internal_kubernetes_deploy/include.yml
-  - .gitlab/kitchen_cleanup/include.yml
-  - .gitlab/kitchen_deploy/kitchen_deploy.yml
-  - .gitlab/kitchen_testing/include.yml
   - .gitlab/lint/include.yml
   - .gitlab/maintenance_jobs/include.yml
   - .gitlab/notify/notify.yml
@@ -79,8 +76,6 @@ stages:
   - package_build
   - packaging
   - pkg_metrics
-  - kitchen_deploy
-  - kitchen_testing
   - container_build
   - container_scan
   - check_deploy
@@ -93,16 +88,15 @@ stages:
   - trigger_release
   - choco_and_install_script_deploy
   - internal_image_deploy
+  - e2e_deploy
   - install_script_testing
   - e2e_pre_test
   - e2e_init
   - e2e
   - e2e_cleanup
   - e2e_k8s
   - e2e_install_packages
-  - kitchen_cleanup
   - functional_test
-  - functional_test_cleanup
   - junit_upload
   - internal_kubernetes_deploy
   - post_rc_build
@@ -124,7 +118,7 @@ variables:
   # avoids accidentally overwriting files when downloading artifacts from
   # both RPM and SUSE rpm jobs.
   OMNIBUS_PACKAGE_DIR_SUSE: $CI_PROJECT_DIR/omnibus/suse/pkg
-  DD_AGENT_TESTING_DIR: $CI_PROJECT_DIR/test/kitchen
+  DD_AGENT_TESTING_DIR: $CI_PROJECT_DIR/test/new-e2e/tests
   STATIC_BINARIES_DIR: bin/static
   DOGSTATSD_BINARIES_DIR: bin/dogstatsd
   AGENT_BINARIES_DIR: bin/agent
@@ -263,8 +257,6 @@ variables:
   E2E_AZURE: e2e-azure  # agent-devx-loops
   E2E_GCP: e2e-gcp  # agent-devx-loops
   GITLAB_TOKEN: gitlab-token  # agent-devx-infra
-  KITCHEN_AWS: kitchen-aws  # agent-devx-loops
-  KITCHEN_AZURE: kitchen-azure  # agent-devx-loops
   INSTALL_SCRIPT_API_KEY_ORG2: install-script-api-key-org-2  # agent-devx-infra
   MACOS_GITHUB_APP_1: macos-github-app-one  # agent-devx-infra
   MACOS_GITHUB_APP_2: macos-github-app-two  # agent-devx-infra
@@ -658,26 +650,12 @@ workflow:
     when: manual
     allow_failure: true
 
-.on_kitchen_tests:
+.on_e2e_tests:
   - <<: *if_installer_tests
 
-.on_kitchen_tests_always:
-  - <<: *if_installer_tests
-    when: always
-
 .on_all_install_script_tests:
   - <<: *if_installer_tests
 
-# Default kitchen tests are also run on dev branches
-# In that case, the target OS versions is a subset of the
-# available versions, stored in DEFAULT_KITCHEN_OSVERS
-.on_default_kitchen_tests:
-  - !reference [.except_mergequeue]
-  - <<: *if_installer_tests
-  - <<: *if_auto_e2e_tests
-    variables:
-      KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS
-
 .on_default_new_e2e_tests:
   - !reference [.except_mergequeue]
   - <<: *if_disable_e2e_tests
@@ -687,15 +665,6 @@ workflow:
     variables:
       E2E_OSVERS: $E2E_BRANCH_OSVERS
 
-.on_default_kitchen_tests_always:
-  - !reference [.except_mergequeue]
-  - <<: *if_installer_tests
-    when: always
-  - <<: *if_auto_e2e_tests
-    when: always
-    variables:
-      KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS
-
 .on_main_or_testing_cleanup:
   - <<: *if_main_branch
   - <<: *if_testing_cleanup
@@ -1186,13 +1155,6 @@ workflow:
         - tasks/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
-.on_kitchen_invoke_tasks_changes:
-  - <<: *if_main_branch
-  - changes:
-      paths:
-        - test/kitchen/tasks/**/*
-      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
-
 .on_powershell_module_or_e2e_changes_or_manual:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:

.gitlab/.ci-linters.yml

@@ -15,15 +15,13 @@ needs-rules:
     - build_dogstatsd_static-binary_x64
     - build_processed_btfhub_archive
     - check_already_deployed_version_7
-    - cleanup_kitchen_functional_test
     - compute_gitlab_ci_config
     - deploy_containers-cws-instrumentation-final-versioned
     - deploy_containers-cws-instrumentation-latest
     - deploy_containers-cws-instrumentation-rc-mutable
     - deploy_containers-cws-instrumentation-rc-versioned
     - dogstatsd_x64_size_test
     - go_mod_tidy_check
-    - kitchen_cleanup_azure-a7
     - lint_flavor_dogstatsd_linux-x64
     - lint_flavor_heroku_linux-x64
     - lint_flavor_iot_linux-x64
@@ -33,9 +31,6 @@ needs-rules:
     - lint_linux-x64
     - lint_macos_gitlab_amd64
     - new-e2e-eks-cleanup-on-failure
-    - periodic_kitchen_cleanup_azure
-    - periodic_kitchen_cleanup_ec2
-    - periodic_kitchen_cleanup_s3
     - publish_winget_7_x64
     - revert_latest_7
     - security_go_generate_check
@@ -81,18 +76,13 @@ job-owners:
     - installer-install-scripts
     - integration_tests_otel
     - invoke_unit_tests
-    - kitchen_cleanup_azure-a7
-    - kitchen_invoke_unit_tests
     - new-e2e-cspm
     - new-e2e-otel
     - new-e2e-package-signing-debian-a7-x86_64
     - new-e2e-package-signing-suse-a7-x86_64
     - new-e2e-unit-tests
     - ot_agent_deb-arm64-a7
     - ot_agent_deb-x64-a7
-    - periodic_kitchen_cleanup_azure
-    - periodic_kitchen_cleanup_ec2
-    - periodic_kitchen_cleanup_s3
     - publish_choco_7_x64
     - publish_fakeintake
     - publish_fakeintake_latest

.gitlab/JOBOWNERS

@@ -74,9 +74,6 @@ deploy_rpm_testing*                  @DataDog/agent-delivery
 deploy_suse_rpm_testing*             @DataDog/agent-delivery
 deploy_windows_testing*              @DataDog/agent-delivery
 
-# Kitchen tests
-kitchen_windows*                     @DataDog/windows-agent
-
 # Image build
 docker_build*                        @DataDog/agent-delivery
 
@@ -133,15 +130,11 @@ trigger_auto_staging_release           @DataDog/agent-delivery
 integration_tests_windows*    @DataDog/windows-agent
 integration_tests_otel        @DataDog/opentelemetry
 docker_image_build_otel       @DataDog/opentelemetry
-ddflare_extension_ocb_build   @DataDog/opentelemetry
+datadog_otel_components_ocb_build  @DataDog/opentelemetry
 agent_integration_tests       @DataDog/container-integrations
 docker_integration_tests      @DataDog/container-integrations
 
 # Functional test
-kitchen_*_system_probe_windows*                @DataDog/windows-kernel-integrations
-kitchen_*_security_agent*                      @DataDog/agent-security
-kitchen_*_process_agent*                       @DataDog/processes
-cleanup_kitchen_functional_test                @DataDog/windows-kernel-integrations @DataDog/agent-security
 serverless_cold_start_performance-deb_x64      @DataDog/serverless
 oracle*                                        @DataDog/database-monitoring
 

.gitlab/check_merge/do_not_merge.yml

@@ -22,5 +22,5 @@ do-not-merge:
             echo "This workflow fails so that the pull request cannot be merged"
             exit 1
           fi
-      - inv -e release.check-omnibus-branches || exit 1
+      - inv -e release.check-omnibus-branches --no-worktree || exit 1
       - exit 0

.gitlab/e2e_install_packages/common.yml

@@ -1,5 +1,5 @@
 .new-e2e_agent_a7:
-  rules: !reference [.on_kitchen_tests] #TODO: Change when migration is complete to another name without 'kitchen'
+  rules: !reference [.on_e2e_tests]
   variables:
     AGENT_MAJOR_VERSION: 7
 

.gitlab/e2e_install_packages/windows.yml

@@ -14,7 +14,7 @@
     - !reference [.new_e2e_template, before_script]
   script:
     # LAST_STABLE_VERSION is used for upgrade test
-    - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::7")
+    - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::7" --no-worktree)
     - !reference [.new_e2e_template, script]
 
 .new-e2e_windows_domain_test:
@@ -28,7 +28,7 @@
     - !reference [.new_e2e_template, before_script]
   script:
     # LAST_STABLE_VERSION is used for upgrade test
-    - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::7")
+    - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::7" --no-worktree)
     - !reference [.new_e2e_template, script]
 
 .new-e2e_windows_installer_v7_tests:

.gitlab/kitchen_deploy/kitchen_deploy.yml → .gitlab/e2e_testing_deploy/e2e_deploy.yml

@@ -1,6 +1,6 @@
 ---
-# kitchen_deploy stage
-# Contains jobs which deploy Agent package to testing repsoitories that are used in kitchen tests.
+# e2e_deploy stage
+# Contains jobs which deploy Agent package to testing repsoitories that are used in e2e tests.
 
 .setup_rpm_signing_key: &setup_rpm_signing_key
   - printf -- "$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $RPM_GPG_KEY)" | gpg --import --batch
@@ -33,7 +33,7 @@
   - popd
 
 .deploy_deb_testing-a7:
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   variables:
@@ -86,7 +86,7 @@ deploy_deb_testing-a7_arm64:
     - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 7 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb
 
 .deploy_rpm_testing-a7:
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   variables:
@@ -132,7 +132,7 @@ deploy_suse_rpm_testing_x64-a7:
     - !reference [.except_no_tests_no_deploy]
     - !reference [.except_mergequeue]
     - when: on_success
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   needs:
@@ -154,10 +154,10 @@ deploy_suse_rpm_testing_x64-a7:
 
 deploy_suse_rpm_testing_arm64-a7:
   rules:
-    - !reference [.on_kitchen_tests]
+    - !reference [.on_e2e_tests]
     - !reference [.on_installer_or_e2e_changes]
     - !reference [.manual]
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   needs: ["installer_suse_rpm-arm64", "agent_suse-arm64-a7", "lint_linux-arm64"]
@@ -175,7 +175,7 @@ deploy_windows_testing-a7:
     - !reference [.except_no_tests_no_deploy]
     - !reference [.except_mergequeue]
     - when: on_success
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   needs: