Merge remote-tracking branch 'origin/main' into mx-psi/otel-mapping-v…

…0.9.0

.github/CODEOWNERS

@@ -28,7 +28,7 @@
 # Todo: is this file still needed?
 /Makefile.trace                         @DataDog/agent-platform
 
-/release.json                           @DataDog/agent-platform @DataDog/agent-shared-components @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations
+/release.json                           @DataDog/agent-platform @DataDog/agent-metrics-logs @DataDog/windows-kernel-integrations
 /requirements.txt                       @DataDog/agent-platform
 /pyproject.toml                         @DataDog/agent-platform
 /setup.cfg                              @DataDog/agent-platform
@@ -306,16 +306,17 @@
 /pkg/collector/corechecks/windows_event_log/ @DataDog/windows-agent
 /pkg/config/                            @DataDog/agent-shared-components
 /pkg/config/config_template.yaml        @DataDog/agent-shared-components @DataDog/documentation
-/pkg/config/apm.go                      @DataDog/agent-apm
+/pkg/config/setup/apm.go                @DataDog/agent-apm
 /pkg/config/autodiscovery/              @Datadog/container-integrations
 /pkg/config/env                         @DataDog/container-integrations @DataDog/container-app
 /pkg/config/logs                        @Datadog/agent-shared-components @Datadog/agent-platform
 /pkg/config/logs/internal/seelog/seelog_config.go          @Datadog/agent-shared-components
-/pkg/config/process*.go                 @DataDog/processes
-/pkg/config/system_probe.go               @DataDog/ebpf-platform
-/pkg/config/system_probe_cws.go           @DataDog/agent-security
-/pkg/config/system_probe_cws_notwin.go    @DataDog/agent-security
-/pkg/config/system_probe_cws_windows.go   @DataDog/windows-kernel-integrations
+/pkg/config/setup                            @DataDog/agent-shared-components
+/pkg/config/setup/process*.go                 @DataDog/processes
+/pkg/config/setup/system_probe.go               @DataDog/ebpf-platform
+/pkg/config/setup/system_probe_cws.go           @DataDog/agent-security
+/pkg/config/setup/system_probe_cws_notwin.go    @DataDog/agent-security
+/pkg/config/setup/system_probe_cws_windows.go   @DataDog/windows-kernel-integrations
 /pkg/config/remote/                     @DataDog/remote-config
 /pkg/config/remote/meta/                @DataDog/remote-config @DataDog/software-integrity-and-trust
 /pkg/containerlifecycle/                @Datadog/container-integrations
@@ -340,7 +341,6 @@
 /pkg/proto/datadog/trace                @DataDog/agent-apm
 /pkg/remoteconfig/                      @DataDog/remote-config
 /pkg/runtime/                           @DataDog/agent-shared-components
-/pkg/secrets/                           @DataDog/agent-shared-components
 /pkg/serializer/                        @DataDog/agent-metrics-logs
 /pkg/tagger/                            @DataDog/container-integrations
 /pkg/tagset/                            @DataDog/agent-shared-components

.gitlab/e2e.yml

@@ -207,13 +207,28 @@ new-e2e-agent-shared-components-main:
   # Temporary, until we manage to stabilize those tests.
   allow_failure: true
 
+.agent-subcommands-tests-matrix: &agent-subcommands-tests-matrix
+  parallel:
+    matrix:
+      - EXTRA_PARAMS: --run TestSubcommandSuite
+      - EXTRA_PARAMS: --run TestAgentSecretSuite
+      - EXTRA_PARAMS: --run TestAgentConfigSuite
+      - EXTRA_PARAMS: --run TestAgentHostnameEC2Suite
+      - EXTRA_PARAMS: --run TestAgentDiagnoseEC2Suite
+      - EXTRA_PARAMS: --run TestAgentConfigCheckSuite
+      - EXTRA_PARAMS: --run TestLinuxFlareSuite
+      - EXTRA_PARAMS: --run TestWindowsFlareSuite
+      - EXTRA_PARAMS: --run TestLinuxSecretSuite
+      - EXTRA_PARAMS: --run TestWindowsSecretSuite
+
 new-e2e-agent-subcommands-dev:
   extends: .new_e2e_template
   rules: !reference [.on_dev_branch_manual]
   needs: []
   variables:
     TARGETS: ./tests/agent-subcommands
     TEAM: agent-shared-components
+  <<: *agent-subcommands-tests-matrix
 
 new-e2e-agent-subcommands-main:
   extends: .new_e2e_template
@@ -224,6 +239,9 @@ new-e2e-agent-subcommands-main:
     TEAM: agent-shared-components
   # Temporary, until we manage to stabilize those tests.
   allow_failure: true
+  <<: *agent-subcommands-tests-matrix
+
+
 
 new-e2e-language-detection-dev:
   extends: .new_e2e_template

.gitlab/e2e_test_junit_upload.yml

@@ -39,6 +39,23 @@ e2e_test_junit_upload:
     - new-e2e-agent-platform-install-script-centos-fips-a7-x86_64
     - new-e2e-agent-platform-install-script-centos-fips-iot-agent-a7-x86_64
     - new-e2e-agent-platform-install-script-centos-fips-dogstatsd-a7-x86_64
+    - new-e2e-agent-platform-step-by-step-debian-a7-x64
+    - new-e2e-agent-platform-step-by-step-debian-a7-arm64
+    - new-e2e-agent-platform-step-by-step-debian-a6-x86_64
+    - new-e2e-agent-platform-step-by-step-debian-a6-arm64
+    - new-e2e-agent-platform-step-by-step-ubuntu-a6-x86_64
+    - new-e2e-agent-platform-step-by-step-ubuntu-a6-arm64
+    - new-e2e-agent-platform-step-by-step-ubuntu-a7-x86_64
+    - new-e2e-agent-platform-step-by-step-ubuntu-a7-arm64
+    - new-e2e-agent-platform-step-by-step-suse-a6-x86_64
+    - new-e2e-agent-platform-step-by-step-suse-a7-x86_64
+    - new-e2e-agent-platform-step-by-step-suse-a7-arm64
+    - new-e2e-agent-platform-step-by-step-centos-a6-x86_64
+    - new-e2e-agent-platform-step-by-step-centos-a7-x86_64
+    - new-e2e-agent-platform-step-by-step-amazonlinux-a6-x86_64
+    - new-e2e-agent-platform-step-by-step-amazonlinux-a6-arm64
+    - new-e2e-agent-platform-step-by-step-amazonlinux-a7-x64
+    - new-e2e-agent-platform-step-by-step-amazonlinux-a7-arm64
     - new-e2e-npm-main
     - new-e2e-aml-main
     - new-e2e-process-main

.gitlab/kitchen_testing/centos.yml

@@ -99,38 +99,6 @@
 # We only want to run step-by-step tests on deploy pipelines,
 # which is why they have a different rule (if_deploy_6/7)
 
-kitchen_centos_step_by_step_agent-a6:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_centos_without_fips_a6
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a6]
-
-kitchen_centos_fips_step_by_step_agent-a6:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_centos_with_fips_a6
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a6]
-
-kitchen_centos_step_by_step_agent-a7:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_centos_without_fips_a7
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a7]
-
-kitchen_centos_fips_step_by_step_agent-a7:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_centos_with_fips_a7
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a7]
-
 # Agent 5 RPMs won't install on CentOS/RHEL 8 in FIPS mode, so we always
 # run upgrade5 on all systems with FIPS off
 kitchen_centos_upgrade5_agent-a6:

.gitlab/kitchen_testing/debian.yml

@@ -85,20 +85,6 @@ kitchen_debian_install_script_heroku_agent-a6:
 # We only want to run step-by-step tests on deploy pipelines,
 # which is why they have a different rule (if_deploy_6/7)
 
-kitchen_debian_step_by_step_agent-a6:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_debian_a6_x64
-    - .kitchen_test_step_by_step_agent
-  rules: !reference [.on_deploy_a6]
-
-kitchen_debian_step_by_step_agent-a7:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_debian_a7_x64
-    - .kitchen_test_step_by_step_agent
-  rules: !reference [.on_deploy_a7]
-
 kitchen_debian_upgrade5_agent-a6:
   extends:
     - .kitchen_scenario_debian_a6_x64

.gitlab/kitchen_testing/suse.yml

@@ -78,22 +78,6 @@ kitchen_suse_install_script_dogstatsd_x64-a7:
 # We only want to run step-by-step tests on deploy pipelines,
 # which is why they have a different rule (if_deploy_6/7)
 
-kitchen_suse_step_by_step_agent_x64-a6:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_suse_x64_a6
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a6]
-
-kitchen_suse_step_by_step_agent_x64-a7:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_suse_x64_a7
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a7]
-
 kitchen_suse_install_script_agent_arm64-a7:
   # Run install script test on branches, on a reduced number of platforms
   rules:

.gitlab/kitchen_testing/ubuntu.yml

@@ -68,38 +68,6 @@
 # We only want to run step-by-step tests on deploy pipelines,
 # which is why they have a different rule (if_deploy_6/7)
 
-kitchen_ubuntu_step_by_step_agent-a6_x64:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_ubuntu_a6_x64
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a6]
-
-kitchen_ubuntu_step_by_step_agent-a6_arm64:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_ubuntu_a6_arm64
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a6]
-
-kitchen_ubuntu_step_by_step_agent-a7_x64:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_ubuntu_a7_x64
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a7]
-
-kitchen_ubuntu_step_by_step_agent-a7_arm64:
-  extends:
-    - .kitchen_os_with_cws
-    - .kitchen_scenario_ubuntu_a7_arm64
-    - .kitchen_test_step_by_step_agent
-  rules:
-    !reference [.on_deploy_a7]
-
 kitchen_ubuntu_upgrade5_agent-a6:
   extends:
     - .kitchen_scenario_ubuntu_a6_x64

.gitlab/new-e2e_common/testing.yml

@@ -14,4 +14,10 @@
   variables:
     TARGETS: ./tests/agent-platform/install-script
     TEAM: agent-platform
+    EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --major-version $AGENT_MAJOR_VERSION --arch $E2E_ARCH --flavor $FLAVOR --no-verbose
+
+.new-e2e_step_by_step:
+  variables:
+    TARGETS: ./tests/agent-platform/step-by-step
+    TEAM: agent-platform
     EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --cws-supported-osversion $E2E_CWS_SUPPORTED_OSVERS --major-version $AGENT_MAJOR_VERSION --arch $E2E_ARCH --flavor $FLAVOR

.gitlab/new-e2e_testing/amazonlinux.yml

@@ -37,7 +37,7 @@
 
 new-e2e-agent-platform-install-script-amazonlinux-a6-x86_64:
   stage: kitchen_testing
-  extends: 
+  extends:
     - .new_e2e_template
     - .new-e2e_install_script
     - .new-e2e_os_amazonlinux
@@ -48,7 +48,7 @@ new-e2e-agent-platform-install-script-amazonlinux-a6-x86_64:
 
 new-e2e-agent-platform-install-script-amazonlinux-a6-arm64:
   stage: kitchen_testing
-  extends: 
+  extends:
     - .new_e2e_template
     - .new-e2e_install_script
     - .new-e2e_os_amazonlinux
@@ -59,7 +59,7 @@ new-e2e-agent-platform-install-script-amazonlinux-a6-arm64:
 
 new-e2e-agent-platform-install-script-amazonlinux-a7-x64:
   stage: kitchen_testing
-  extends: 
+  extends:
     - .new_e2e_template
     - .new-e2e_install_script
     - .new-e2e_os_amazonlinux
@@ -72,7 +72,7 @@ new-e2e-agent-platform-install-script-amazonlinux-a7-x64:
 
 new-e2e-agent-platform-install-script-amazonlinux-a7-arm64:
   stage: kitchen_testing
-  extends: 
+  extends:
     - .new_e2e_template
     - .new-e2e_install_script
     - .new-e2e_os_amazonlinux
@@ -82,3 +82,56 @@ new-e2e-agent-platform-install-script-amazonlinux-a7-arm64:
     !reference [.on_all_new-e2e_tests_a7]
   variables:
     FLAVOR: datadog-agent
+
+
+new-e2e-agent-platform-step-by-step-amazonlinux-a6-x86_64:
+  stage: kitchen_testing
+  extends:
+    - .new_e2e_template
+    - .new-e2e_step_by_step
+    - .new-e2e_os_amazonlinux
+    - .new-e2e_amazonlinux_a6_x86_64
+    - .new-e2e_agent_a6
+  rules:
+    !reference [.on_deploy_a6]
+  variables:
+    FLAVOR: datadog-agent
+
+new-e2e-agent-platform-step-by-step-amazonlinux-a6-arm64:
+  stage: kitchen_testing
+  extends:
+    - .new_e2e_template
+    - .new-e2e_step_by_step
+    - .new-e2e_os_amazonlinux
+    - .new-e2e_amazonlinux_a6_arm64
+    - .new-e2e_agent_a6
+  rules:
+    !reference [.on_deploy_a6]
+  variables:
+    FLAVOR: datadog-agent
+
+new-e2e-agent-platform-step-by-step-amazonlinux-a7-x64:
+  stage: kitchen_testing
+  extends:
+    - .new_e2e_template
+    - .new-e2e_step_by_step
+    - .new-e2e_os_amazonlinux
+    - .new-e2e_amazonlinux_a7_x86_64
+    - .new-e2e_agent_a7
+  rules:
+    !reference [.on_deploy_a7]
+  variables:
+    FLAVOR: datadog-agent
+
+new-e2e-agent-platform-step-by-step-amazonlinux-a7-arm64:
+  stage: kitchen_testing
+  extends:
+    - .new_e2e_template
+    - .new-e2e_step_by_step
+    - .new-e2e_os_amazonlinux
+    - .new-e2e_amazonlinux_a7_arm64
+    - .new-e2e_agent_a7
+  rules:
+    !reference [.on_deploy_a7]
+  variables:
+    FLAVOR: datadog-agent