Merge branch 'main' of github.com:DataDog/datadog-agent into olivierg…

…/create-orchestrator-fwd

.ddqa/config.toml

@@ -30,7 +30,10 @@ jira_statuses = [
 ]
 github_team = "agent-shared-components"
 github_labels = ["team/agent-shared-components"]
-exclude_members = ["olivielpeau", "sgnn7"]
+exclude_members = [
+  "sgnn7",
+  "truthbk",
+]
 
 [teams."Agent Platform"]
 jira_project = "APL"
@@ -262,3 +265,14 @@ jira_statuses = [
 ]
 github_team = "apm-onboarding"
 github_labels = ["team/apm-onboarding"]
+
+[teams."Agent Release Management"]
+jira_project = "AGNTR"
+jira_issue_type = "Task"
+jira_statuses = [
+  "To Do",
+  "In Progress",
+  "Done",
+]
+github_team = "agent-release-management"
+github_labels = ["team/agent-release-management"]

.github/CODEOWNERS

@@ -83,6 +83,8 @@
 /.gitlab/functional_test/security_agent.yml          @DataDog/agent-security @DataDog/agent-platform
 /.gitlab/functional_test/serverless.yml              @DataDog/serverless @DataDog/agent-platform
 /.gitlab/functional_test_cleanup.yml                 @DataDog/agent-security @DataDog/ebpf-platform @DataDog/agent-platform
+/.gitlab/functional_test/system_probe_windows.yml    @DataDog/agent-platform @DataDog/windows-kernel-integrations
+/.gitlab/functional_test/common.yml                  @DataDog/agent-platform @DataDog/windows-kernel-integrations @DataDog/ebpf-platform
 /.gitlab/functional_test_sysprobe/system_probe.yml   @DataDog/ebpf-platform @DataDog/agent-platform
 
 /.gitlab/integration_test/windows.yml                @DataDog/agent-platform @DataDog/windows-agent
@@ -234,6 +236,7 @@
 /pkg/aggregator/                        @DataDog/agent-metrics-logs
 /pkg/collector/                         @DataDog/agent-metrics-logs
 /pkg/cli/                               @DataDog/agent-shared-components
+/pkg/cli/subcommands/clusterchecks      @DataDog/container-integrations
 /pkg/dogstatsd/                         @DataDog/agent-metrics-logs
 /pkg/errors/                            @DataDog/agent-shared-components
 /pkg/forwarder/                         @DataDog/agent-metrics-logs @DataDog/agent-shared-components
@@ -282,14 +285,13 @@
 /pkg/collector/corechecks/embed/process_agent*.go  @Datadog/agent-platform @DataDog/processes
 /pkg/collector/corechecks/orchestrator/   @DataDog/container-app
 /pkg/collector/corechecks/kubernetes/       @DataDog/container-integrations
-/pkg/collector/corechecks/net/          @DataDog/agent-platform
+/pkg/collector/corechecks/net/          @DataDog/platform-integrations
 /pkg/collector/corechecks/oracle-dbm        @DataDog/database-monitoring
 /pkg/collector/corechecks/sbom/         @DataDog/container-integrations
 /pkg/collector/corechecks/snmp/         @DataDog/network-device-monitoring
-/pkg/collector/corechecks/system/                 @DataDog/agent-platform
+/pkg/collector/corechecks/system/                 @DataDog/platform-integrations
+/pkg/collector/corechecks/system/**/*_windows*.go @DataDog/platform-integrations @DataDog/windows-agent
 /pkg/collector/corechecks/system/wincrashdetect/  @DataDog/windows-kernel-integrations
-/pkg/collector/corechecks/system/**/*_windows*.go @DataDog/agent-platform @DataDog/windows-agent
-/pkg/collector/corechecks/system/wincrashdetect/         @DataDog/windows-kernel-integrations
 /pkg/collector/corechecks/system/winkmem/         @DataDog/windows-agent
 /pkg/collector/corechecks/system/winproc/         @DataDog/windows-agent
 /pkg/collector/corechecks/systemd/      @DataDog/agent-integrations
@@ -355,6 +357,8 @@
 /pkg/util/cgroups/                      @DataDog/container-integrations
 /pkg/util/retry/                        @DataDog/container-integrations
 /pkg/util/intern/                       @DataDog/ebpf-platform
+/pkg/util/crashreport/                  @DataDog/windows-kernel-integrations
+/pkg/util/pdhutil/                      @DataDog/windows-agent
 /pkg/util/winutil/                      @DataDog/windows-agent
 /pkg/languagedetection                  @DataDog/processes @DataDog/universal-service-monitoring
 /pkg/logs/                              @DataDog/agent-metrics-logs
@@ -481,6 +485,7 @@
 /test/new-e2e/tests/language-detection        @DataDog/processes
 /test/new-e2e/tests/ndm                       @DataDog/network-device-monitoring
 /test/new-e2e/tests/npm                       @DataDog/Networks
+/test/new-e2e/tests/cws                       @DataDog/agent-security
 /test/new-e2e/tests/agent-platform            @DataDog/agent-platform
 /test/system/                                 @DataDog/agent-shared-components
 /test/system/dogstatsd/                       @DataDog/agent-metrics-logs

.github/workflows/slapr.yml

@@ -14,6 +14,10 @@ jobs:
         channel_variables:
           - SLACK_AGENT_PLATFORM_ID
           - SLACK_AGENT_PLATFORM_REVIEWS_ID
+          - SLACK_OPENTELEMETRY_ID
+          - SLACK_AGENT_METRICS_LOGS_ID
+          - SLACK_CONTAINER_INTEGRATIONS_ID
+          - SLACK_AGENT_SHARED_COMPONENTS_ID
     steps:
       - uses: DataDog/slapr@master
         env:
@@ -22,7 +26,7 @@ jobs:
           SLACK_API_TOKEN: "${{ secrets.SLACK_API_TOKEN }}"
           SLAPR_BOT_USER_ID: "${{ secrets.SLAPR_BOT_USER_ID }}"
           SLAPR_EMOJI_REVIEW_STARTED: "review_started"
-          SLAPR_EMOJI_APPROVED: "approved2"
-          SLAPR_EMOJI_CHANGES_REQUESTED: "changes_requested"
+          SLAPR_EMOJI_APPROVED: "gh-approved"
+          SLAPR_EMOJI_CHANGES_REQUESTED: "request-changes"
           SLAPR_EMOJI_MERGED: "merged"
           SLAPR_EMOJI_CLOSED: "closed"

.gitlab-ci.yml

@@ -45,6 +45,7 @@ include:
   - /.gitlab/notify.yml
   - /.gitlab/kitchen_common/cleanup.yml
   - /.gitlab/kitchen_common/testing.yml
+  - /.gitlab/functional_test/common.yml
   - /.gitlab/benchmarks/benchmarks.yml
   - /.gitlab/benchmarks/macrobenchmarks.yml
   - /.gitlab/rc_kubernetes_deploy.yml
@@ -176,7 +177,7 @@ variables:
   # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 2edaabddf28b
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: a60850d5606e
   DATADOG_AGENT_BUILDERS: v22276738-b36b132
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
@@ -952,6 +953,7 @@ workflow:
         - test/kitchen/site-cookbooks/dd-system-probe-check/**/*
         - test/kitchen/test/integration/system-probe-test/**/*
         - test/kitchen/test/integration/win-sysprobe-test/**/*
+        - .gitlab/functional_test/system_probe_windows.yml
         - .gitlab/functional_test_sysprobe/system_probe.yml
         - .gitlab/kernel_version_testing/system_probe.yml
         - test/new-e2e/system-probe/**/*
@@ -1001,4 +1003,4 @@ workflow:
 
 .except_mergequeue:
   - <<: *if_mergequeue
-    when: never
+    when: never

.gitlab/e2e.yml

@@ -259,6 +259,16 @@ new-e2e-cws-dev:
   needs: []
   variables:
     TARGETS: ./tests/cws
+    TEAM: csm-threats-agent
+  # Temporary, until we manage to stabilize those tests.
+  allow_failure: true
+
+new-e2e-cws-main:
+  extends: .new_e2e_template
+  rules: !reference [.on_main_and_no_skip_e2e]
+  variables:
+    TARGETS: ./tests/cws
+    TEAM: csm-threats-agent
   # Temporary, until we manage to stabilize those tests.
   allow_failure: true
 

.gitlab/e2e_test_junit_upload.yml

@@ -14,9 +14,17 @@ e2e_test_junit_upload:
     - new-e2e-agent-shared-components-main
     - new-e2e-agent-subcommands-main
     - new-e2e-language-detection-main
-    - new-e2e-agent-platform-install-script-debian-a7-x64
+    - new-e2e-agent-platform-install-script-debian-a7-x86_64
     - new-e2e-agent-platform-install-script-debian-a7-arm64
+    - new-e2e-agent-platform-install-script-debian-iot-agent-a7-x86_64
+    - new-e2e-agent-platform-install-script-debian-dogstatsd-a7-x86_64
+    - new-e2e-agent-platform-install-script-debian-heroku-agent-a7-x86_64
+    - new-e2e-agent-platform-install-script-amazonlinux-a6-x86_64
+    - new-e2e-agent-platform-install-script-amazonlinux-a6-arm64
+    - new-e2e-agent-platform-install-script-amazonlinux-a7-x64
+    - new-e2e-agent-platform-install-script-amazonlinux-a7-arm64
     - new-e2e-npm-main
+    - new-e2e-cws-main
   script:
     - set +x
     - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)

.gitlab/functional_test.yml

@@ -7,5 +7,6 @@ include:
   - /.gitlab/functional_test/serverless.yml
   - /.gitlab/functional_test/regression_detector.yml
   - /.gitlab/functional_test/workload_checks.yml
+  - /.gitlab/functional_test/system_probe_windows.yml
   - /.gitlab/kernel_version_testing/system_probe.yml
   - /.gitlab/functional_test_sysprobe/system_probe.yml

.gitlab/functional_test/common.yml

@@ -0,0 +1,18 @@
+---
+# FIXME: our current Gitlab version doesn't support importing a file more than once
+# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file
+# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987
+# include:
+#   - /.gitlab/kitchen_common/testing.yml
+
+.kitchen_test_system_probe:
+  extends:
+    - .kitchen_common
+    - .kitchen_datadog_agent_flavor
+  rules:
+    !reference [.on_system_probe_changes_or_manual]
+  timeout: 3h
+  variables:
+    AGENT_MAJOR_VERSION: 7
+    DD_PIPELINE_ID: $CI_PIPELINE_ID-fnct
+    CHEF_VERSION: 14.15.6

.gitlab/functional_test/system_probe_windows.yml

@@ -0,0 +1,29 @@
+---
+# FIXME: our current Gitlab version doesn't support importing a file more than once
+# For now, the workaround is to include "common" files once in the top-level .gitlab-ci.yml file
+# See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987
+# include:
+#   - /.gitlab/kitchen_common/testing.yml
+#   - /.gitlab/functional_test/common.yml
+
+kitchen_test_system_probe_windows_x64:
+  extends:
+    - .kitchen_agent_a7
+    - .kitchen_os_windows
+    - .kitchen_test_system_probe
+    - .kitchen_azure_x64
+    - .kitchen_azure_location_north_central_us
+  stage: functional_test
+  needs: [ "tests_windows_sysprobe_x64" ]
+  variables:
+    KITCHEN_ARCH: x86_64
+    KITCHEN_OSVERS: "win2016"
+    CHEF_VERSION: 14.12.9 # newer versions error out during kitchen setup of azure VM
+  before_script:
+    - export WINDOWS_DDNPM_DRIVER=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_DRIVER")
+    - export WINDOWS_DDNPM_VERSION=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_VERSION")
+    - export WINDOWS_DDNPM_SHASUM=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_SHASUM")
+    - pushd $DD_AGENT_TESTING_DIR
+    - tasks/kitchen_setup.sh
+  script:
+    - tasks/run-test-kitchen.sh windows-sysprobe-test $AGENT_MAJOR_VERSION

.gitlab/functional_test_sysprobe/system_probe.yml

@@ -4,23 +4,12 @@
 # See: https://gitlab.com/gitlab-org/gitlab/-/issues/28987
 # include:
 #   - /.gitlab/kitchen_common/testing.yml
-
-.kitchen_test_system_probe:
-  extends:
-    - .kitchen_common
-    - .kitchen_datadog_agent_flavor
-  rules:
-    !reference [.on_system_probe_changes_or_manual]
-  stage: functional_test_sysprobe
-  timeout: 3h
-  variables:
-    AGENT_MAJOR_VERSION: 7
-    DD_PIPELINE_ID: $CI_PIPELINE_ID-fnct
-    CHEF_VERSION: 14.15.6
+#   - /.gitlab/functional_test/common.yml
 
 .kitchen_test_system_probe_linux:
   extends:
     - .kitchen_test_system_probe
+  stage: functional_test_sysprobe
   before_script:
     - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/job_env.txt
     - echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-system-probe-check/files/job_env.txt
@@ -180,24 +169,3 @@ kitchen_test_system_probe_linux_arm64:
         KITCHEN_OSVERS: "ubuntu-23-04"
       - KITCHEN_PLATFORM: "ubuntu"
         KITCHEN_OSVERS: "ubuntu-23-10"
-
-kitchen_test_system_probe_windows_x64:
-  extends:
-    - .kitchen_agent_a7
-    - .kitchen_os_windows
-    - .kitchen_test_system_probe
-    - .kitchen_azure_x64
-    - .kitchen_azure_location_north_central_us
-  needs: [ "tests_windows_sysprobe_x64" ]
-  variables:
-    KITCHEN_ARCH: x86_64
-    KITCHEN_OSVERS: "win2016"
-    CHEF_VERSION: 14.12.9 # newer versions error out during kitchen setup of azure VM
-  before_script:
-    - export WINDOWS_DDNPM_DRIVER=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_DRIVER")
-    - export WINDOWS_DDNPM_VERSION=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_VERSION")
-    - export WINDOWS_DDNPM_SHASUM=$(inv release.get-release-json-value "$RELEASE_VERSION_7::WINDOWS_DDNPM_SHASUM")
-    - pushd $DD_AGENT_TESTING_DIR
-    - tasks/kitchen_setup.sh
-  script:
-    - tasks/run-test-kitchen.sh windows-sysprobe-test $AGENT_MAJOR_VERSION

.gitlab/kernel_version_testing/system_probe.yml

@@ -401,7 +401,7 @@ kernel_matrix_testing_run_tests_x64:
     ARCH: "x86_64"
   parallel:
     matrix:
-      - TAG: ["ubuntu_16.04", "ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79"]
+      - TAG: ["ubuntu_16.04", "ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79", "centos_8"]
 
 kernel_matrix_testing_run_tests_arm64:
   extends:
@@ -414,7 +414,7 @@ kernel_matrix_testing_run_tests_arm64:
     ARCH: "arm64"
   parallel:
     matrix:
-      - TAG: ["ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79"]
+      - TAG: ["ubuntu_18.04", "ubuntu_20.04", "ubuntu_22.04", "ubuntu_23.10", "amzn_4.14", "amzn_5.4", "amzn_5.10", "fedora_37", "fedora_38", "debian_10", "debian_11", "debian_12", "centos_79", "centos_8"]
 
 .kernel_matrix_testing_cleanup:
   stage: kernel_matrix_testing

.gitlab/kitchen_testing.yml

@@ -3,7 +3,6 @@
 # Contains jobs which run kitchen tests on the Agent packages.
 
 include:
-  - /.gitlab/kitchen_testing/amazonlinux.yml
   - /.gitlab/kitchen_testing/centos.yml
   - /.gitlab/kitchen_testing/debian.yml
   - /.gitlab/kitchen_testing/suse.yml