Delete test/kitchen folder (DataDog#31525)

.github/CODEOWNERS

@@ -75,6 +75,7 @@
 /.gitlab/deploy*/*                                   @DataDog/agent-delivery
 /.gitlab/deps_fetch/*                                @DataDog/agent-devx-infra
 /.gitlab/e2e/*                                       @DataDog/agent-devx-infra @DataDog/agent-devx-loops
+/.gitlab/e2e_testing_deploy/*                        @DataDog/agent-devx-loops @DataDog/agent-devx-infra
 /.gitlab/e2e_install_packages/*                      @DataDog/agent-delivery
 /.gitlab/e2e_pre_test/*                              @DataDog/agent-devx-infra @DataDog/agent-devx-loops
 /.gitlab/kernel_matrix_testing/*                     @DataDog/agent-devx-infra @DataDog/ebpf-platform
@@ -90,7 +91,6 @@
 /.gitlab/binary_build/include.yml                    @DataDog/agent-devx-infra
 /.gitlab/binary_build/linux.yml                      @DataDog/agent-devx-infra @DataDog/agent-delivery
 /.gitlab/functional_test/include.yml                 @DataDog/agent-devx-infra
-/.gitlab/functional_test_cleanup/functional_test_cleanup.yml        @DataDog/agent-devx-infra @DataDog/agent-security
 /.gitlab/install_script_testing/install_script_testing.yml          @DataDog/agent-delivery
 /.gitlab/integration_test/dogstatsd.yml              @DataDog/agent-devx-infra @DataDog/agent-metrics-logs
 /.gitlab/integration_test/include.yml                @DataDog/agent-devx-infra
@@ -131,8 +131,6 @@
 
 /.gitlab/deps_build/                                 @DataDog/ebpf-platform @DataDog/agent-delivery @DataDog/windows-agent
 
-/.gitlab/kitchen_*                                   @DataDog/container-ecosystems @DataDog/agent-delivery
-/.gitlab/kitchen_testing/windows.yml                 @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
 /.gitlab/e2e_install_packages/windows.yml            @DataDog/container-ecosystems @DataDog/agent-delivery @DataDog/windows-agent
 
 /.gitlab/common/                                     @DataDog/agent-devx-infra
@@ -144,7 +142,6 @@
 /.gitlab/binary_build/fakeintake.yml                 @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 
 /.gitlab/functional_test/serverless.yml              @DataDog/serverless @Datadog/serverless-aws @DataDog/agent-devx-infra
-/.gitlab/functional_test_cleanup                     @DataDog/agent-security @DataDog/windows-kernel-integrations @DataDog/agent-devx-infra
 /.gitlab/functional_test/oracle.yml                  @DataDog/agent-devx-infra @DataDog/database-monitoring
 
 /.gitlab/powershell_script_deploy                    @DataDog/agent-delivery @DataDog/windows-agent
@@ -590,18 +587,6 @@
 /test/integration/docker/otel_agent_build_tests.py @DataDog/opentelemetry
 /test/integration/serverless            @DataDog/serverless @Datadog/serverless-aws
 /test/integration/serverless_perf       @DataDog/serverless @Datadog/serverless-aws
-/test/kitchen/                          @DataDog/agent-devx-loops
-/test/kitchen/test-definitions/         @DataDog/container-ecosystems @DataDog/agent-delivery
-/test/kitchen/test/integration/         @DataDog/container-ecosystems @DataDog/agent-delivery
-/test/kitchen/site-cookbooks/dd-system-probe-check/ @DataDog/windows-kernel-integrations
-/test/kitchen/test/integration/win-all-subservices/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-alt-dir/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-install-fail/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-installopts/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-no-subservices/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-sysprobe-test/ @DataDog/windows-kernel-integrations
-/test/kitchen/test/integration/win-repair/ @DataDog/windows-agent
-/test/kitchen/test/integration/win-user/ @DataDog/windows-agent
 /test/fakeintake/                             @DataDog/agent-e2e-testing @DataDog/agent-devx-loops
 /test/fakeintake/aggregator/ndmflowAggregator.go @DataDog/ndm-integrations
 /test/fakeintake/aggregator/ndmflowAggregator_test.go @DataDog/ndm-integrations

.gitlab-ci.yml

@@ -24,15 +24,12 @@ include:
   - .gitlab/e2e/e2e.yml
   - .gitlab/e2e_install_packages/include.yml
   - .gitlab/e2e_pre_test/e2e_pre_test.yml
+  - .gitlab/e2e_testing_deploy/e2e_deploy.yml
   - .gitlab/functional_test/include.yml
-  - .gitlab/functional_test_cleanup/functional_test_cleanup.yml
   - .gitlab/install_script_testing/install_script_testing.yml
   - .gitlab/integration_test/include.yml
   - .gitlab/internal_image_deploy/internal_image_deploy.yml
   - .gitlab/internal_kubernetes_deploy/include.yml
-  - .gitlab/kitchen_cleanup/include.yml
-  - .gitlab/kitchen_deploy/kitchen_deploy.yml
-  - .gitlab/kitchen_testing/include.yml
   - .gitlab/lint/include.yml
   - .gitlab/maintenance_jobs/include.yml
   - .gitlab/notify/notify.yml
@@ -79,8 +76,6 @@ stages:
   - package_build
   - packaging
   - pkg_metrics
-  - kitchen_deploy
-  - kitchen_testing
   - container_build
   - container_scan
   - check_deploy
@@ -93,16 +88,15 @@ stages:
   - trigger_release
   - choco_and_install_script_deploy
   - internal_image_deploy
+  - e2e_deploy
   - install_script_testing
   - e2e_pre_test
   - e2e_init
   - e2e
   - e2e_cleanup
   - e2e_k8s
   - e2e_install_packages
-  - kitchen_cleanup
   - functional_test
-  - functional_test_cleanup
   - junit_upload
   - internal_kubernetes_deploy
   - post_rc_build
@@ -124,7 +118,7 @@ variables:
   # avoids accidentally overwriting files when downloading artifacts from
   # both RPM and SUSE rpm jobs.
   OMNIBUS_PACKAGE_DIR_SUSE: $CI_PROJECT_DIR/omnibus/suse/pkg
-  DD_AGENT_TESTING_DIR: $CI_PROJECT_DIR/test/kitchen
+  DD_AGENT_TESTING_DIR: $CI_PROJECT_DIR/test/new-e2e/tests
   STATIC_BINARIES_DIR: bin/static
   DOGSTATSD_BINARIES_DIR: bin/dogstatsd
   AGENT_BINARIES_DIR: bin/agent
@@ -263,8 +257,6 @@ variables:
   E2E_AZURE: e2e-azure  # agent-devx-loops
   E2E_GCP: e2e-gcp  # agent-devx-loops
   GITLAB_TOKEN: gitlab-token  # agent-devx-infra
-  KITCHEN_AWS: kitchen-aws  # agent-devx-loops
-  KITCHEN_AZURE: kitchen-azure  # agent-devx-loops
   INSTALL_SCRIPT_API_KEY_ORG2: install-script-api-key-org-2  # agent-devx-infra
   MACOS_GITHUB_APP_1: macos-github-app-one  # agent-devx-infra
   MACOS_GITHUB_APP_2: macos-github-app-two  # agent-devx-infra
@@ -658,26 +650,12 @@ workflow:
     when: manual
     allow_failure: true
 
-.on_kitchen_tests:
+.on_e2e_tests:
   - <<: *if_installer_tests
 
-.on_kitchen_tests_always:
-  - <<: *if_installer_tests
-    when: always
-
 .on_all_install_script_tests:
   - <<: *if_installer_tests
 
-# Default kitchen tests are also run on dev branches
-# In that case, the target OS versions is a subset of the
-# available versions, stored in DEFAULT_KITCHEN_OSVERS
-.on_default_kitchen_tests:
-  - !reference [.except_mergequeue]
-  - <<: *if_installer_tests
-  - <<: *if_auto_e2e_tests
-    variables:
-      KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS
-
 .on_default_new_e2e_tests:
   - !reference [.except_mergequeue]
   - <<: *if_disable_e2e_tests
@@ -687,15 +665,6 @@ workflow:
     variables:
       E2E_OSVERS: $E2E_BRANCH_OSVERS
 
-.on_default_kitchen_tests_always:
-  - !reference [.except_mergequeue]
-  - <<: *if_installer_tests
-    when: always
-  - <<: *if_auto_e2e_tests
-    when: always
-    variables:
-      KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS
-
 .on_main_or_testing_cleanup:
   - <<: *if_main_branch
   - <<: *if_testing_cleanup
@@ -1186,13 +1155,6 @@ workflow:
         - tasks/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
-.on_kitchen_invoke_tasks_changes:
-  - <<: *if_main_branch
-  - changes:
-      paths:
-        - test/kitchen/tasks/**/*
-      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
-
 .on_powershell_module_or_e2e_changes_or_manual:
   - !reference [.on_e2e_main_release_or_rc]
   - changes:

.gitlab/.ci-linters.yml

@@ -15,15 +15,13 @@ needs-rules:
     - build_dogstatsd_static-binary_x64
     - build_processed_btfhub_archive
     - check_already_deployed_version_7
-    - cleanup_kitchen_functional_test
     - compute_gitlab_ci_config
     - deploy_containers-cws-instrumentation-final-versioned
     - deploy_containers-cws-instrumentation-latest
     - deploy_containers-cws-instrumentation-rc-mutable
     - deploy_containers-cws-instrumentation-rc-versioned
     - dogstatsd_x64_size_test
     - go_mod_tidy_check
-    - kitchen_cleanup_azure-a7
     - lint_flavor_dogstatsd_linux-x64
     - lint_flavor_heroku_linux-x64
     - lint_flavor_iot_linux-x64
@@ -33,9 +31,6 @@ needs-rules:
     - lint_linux-x64
     - lint_macos_gitlab_amd64
     - new-e2e-eks-cleanup-on-failure
-    - periodic_kitchen_cleanup_azure
-    - periodic_kitchen_cleanup_ec2
-    - periodic_kitchen_cleanup_s3
     - publish_winget_7_x64
     - revert_latest_7
     - security_go_generate_check
@@ -81,18 +76,13 @@ job-owners:
     - installer-install-scripts
     - integration_tests_otel
     - invoke_unit_tests
-    - kitchen_cleanup_azure-a7
-    - kitchen_invoke_unit_tests
     - new-e2e-cspm
     - new-e2e-otel
     - new-e2e-package-signing-debian-a7-x86_64
     - new-e2e-package-signing-suse-a7-x86_64
     - new-e2e-unit-tests
     - ot_agent_deb-arm64-a7
     - ot_agent_deb-x64-a7
-    - periodic_kitchen_cleanup_azure
-    - periodic_kitchen_cleanup_ec2
-    - periodic_kitchen_cleanup_s3
     - publish_choco_7_x64
     - publish_fakeintake
     - publish_fakeintake_latest

.gitlab/JOBOWNERS

@@ -74,9 +74,6 @@ deploy_rpm_testing*                  @DataDog/agent-delivery
 deploy_suse_rpm_testing*             @DataDog/agent-delivery
 deploy_windows_testing*              @DataDog/agent-delivery
 
-# Kitchen tests
-kitchen_windows*                     @DataDog/windows-agent
-
 # Image build
 docker_build*                        @DataDog/agent-delivery
 
@@ -138,10 +135,6 @@ agent_integration_tests       @DataDog/container-integrations
 docker_integration_tests      @DataDog/container-integrations
 
 # Functional test
-kitchen_*_system_probe_windows*                @DataDog/windows-kernel-integrations
-kitchen_*_security_agent*                      @DataDog/agent-security
-kitchen_*_process_agent*                       @DataDog/processes
-cleanup_kitchen_functional_test                @DataDog/windows-kernel-integrations @DataDog/agent-security
 serverless_cold_start_performance-deb_x64      @DataDog/serverless
 oracle*                                        @DataDog/database-monitoring
 

.gitlab/e2e_install_packages/common.yml

@@ -1,5 +1,5 @@
 .new-e2e_agent_a7:
-  rules: !reference [.on_kitchen_tests] #TODO: Change when migration is complete to another name without 'kitchen'
+  rules: !reference [.on_e2e_tests]
   variables:
     AGENT_MAJOR_VERSION: 7
 

.gitlab/kitchen_deploy/kitchen_deploy.yml → .gitlab/e2e_testing_deploy/e2e_deploy.yml

@@ -1,6 +1,6 @@
 ---
-# kitchen_deploy stage
-# Contains jobs which deploy Agent package to testing repsoitories that are used in kitchen tests.
+# e2e_deploy stage
+# Contains jobs which deploy Agent package to testing repsoitories that are used in e2e tests.
 
 .setup_rpm_signing_key: &setup_rpm_signing_key
   - printf -- "$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $RPM_GPG_KEY)" | gpg --import --batch
@@ -33,7 +33,7 @@
   - popd
 
 .deploy_deb_testing-a7:
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   variables:
@@ -86,7 +86,7 @@ deploy_deb_testing-a7_arm64:
     - echo "$APT_SIGNING_KEY_PASSPHRASE" | deb-s3 upload -c "pipeline-$DD_PIPELINE_ID-arm64" -m 7 -b $DEB_TESTING_S3_BUCKET -a arm64 --sign=$DEB_GPG_KEY_ID --gpg_options="--passphrase-fd 0 --batch --digest-algo SHA512" --preserve_versions --visibility public $OMNIBUS_PACKAGE_DIR/datadog-signing-keys_${DD_PIPELINE_ID}.deb
 
 .deploy_rpm_testing-a7:
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   variables:
@@ -132,7 +132,7 @@ deploy_suse_rpm_testing_x64-a7:
     - !reference [.except_no_tests_no_deploy]
     - !reference [.except_mergequeue]
     - when: on_success
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   needs:
@@ -154,10 +154,10 @@ deploy_suse_rpm_testing_x64-a7:
 
 deploy_suse_rpm_testing_arm64-a7:
   rules:
-    - !reference [.on_kitchen_tests]
+    - !reference [.on_e2e_tests]
     - !reference [.on_installer_or_e2e_changes]
     - !reference [.manual]
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   needs: ["installer_suse_rpm-arm64", "agent_suse-arm64-a7", "lint_linux-arm64"]
@@ -175,7 +175,7 @@ deploy_windows_testing-a7:
     - !reference [.except_no_tests_no_deploy]
     - !reference [.except_mergequeue]
     - when: on_success
-  stage: kitchen_deploy
+  stage: e2e_deploy
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   needs:

.gitlab/functional_test/include.yml

@@ -1,6 +1,6 @@
 ---
 # functional_test stage
-# Contains jobs which run kitchen tests on the security-agent and on system-probe
+# Contains jobs which run e2e tests on the security-agent and on system-probe
 
 include:
   - .gitlab/functional_test/serverless.yml

.gitlab/kernel_matrix_testing/common.yml

@@ -1,3 +1,11 @@
+# KMT: EC2 Locations
+.kmt_ec2_location_us_east_1:
+  variables:
+    KITCHEN_EC2_REGION: us-east-1
+    KITCHEN_EC2_SUBNET: subnet-05d7c6b1b5cfea811
+    KITCHEN_EC2_SG_IDS: sg-019917348cb0eb7e7
+
+
 # --- Common scripts
 .shared_filters_and_queries:
   - FILTER_TEAM="Name=tag:team,Values=ebpf-platform"
@@ -122,7 +130,7 @@
 # -- Environment setup
 .kmt_setup_env:
   extends:
-    - .kitchen_ec2_location_us_east_1
+    - .kmt_ec2_location_us_east_1
   stage: kernel_matrix_testing_prepare
   image: registry.ddbuild.io/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
   needs: ["go_deps", "go_tools_deps"]