Merge remote-tracking branch 'origin/main' into hasan.mahmood/sysprob…

…e-pprof-test

.github/CODEOWNERS

@@ -166,7 +166,6 @@
 /cmd/agent/dist/conf.d/snmp.d/          @DataDog/network-device-monitoring
 /cmd/agent/dist/conf.d/win32_event_log.d/ @DataDog/windows-agent
 /cmd/agent/install*.sh                  @DataDog/agent-build-and-releases
-/cmd/agentless-scanner/                 @DataDog/agent-cspm
 /cmd/cluster-agent/                     @DataDog/container-platform
 /cmd/cluster-agent/commands/            @DataDog/container-platform
 /cmd/cluster-agent-cloudfoundry/        @DataDog/platform-integrations

.github/workflows/docs-dev.yml

@@ -6,11 +6,13 @@ on:
     - main
     paths:
     - docs/**
+    - .github/workflows/docs-dev.yml
   pull_request:
     branches:
     - main
     paths:
     - docs/**
+    - .github/workflows/docs-dev.yml
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/send-failure-summary-notification.yml

@@ -0,0 +1,34 @@
+name: Send failure summary notification
+
+on:
+  workflow_dispatch:
+    inputs:
+      # See notify_tests.py to see the format
+      job_failures:
+        description: 'Job failures'
+        required: true
+        type: string
+
+jobs:
+  send-failure-summary-team-notification:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: main
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: pip install -r requirements.txt -r tasks/requirements.txt
+      - name: Create and send failure summary notification to team channel
+        id: send_failure_summary_team_notification
+        env:
+          # Too much data to be passed by arguments
+          JOB_FAILURES: ${{ inputs.job_failures }}
+          SLACK_API_TOKEN: ${{ secrets.SLACK_DATADOG_AGENT_BOT_TOKEN }}
+        run: inv -e notify.send-failure-summary-notification

.gitignore

@@ -37,6 +37,10 @@ Dockerfiles/cluster-agent/nosys-seccomp
 # specific pre-commit hooks
 .pre-commit-config-*.yaml
 
+# Utility tools
+devagent
+deva
+
 # go-generated files
 datadog.yaml
 system-probe.yaml

.gitlab/binary_build/linux.yml

@@ -53,9 +53,8 @@ build_dogstatsd-binary_x64:
 
 build_dogstatsd-binary_arm64:
   rules:
-    - !reference [.on_all_builds]
-    - !reference [.on_packaging_change]
-    - !reference [.on_go-version_change]
+    - !reference [.except_mergequeue]
+    - when: on_success
   stage: binary_build
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES
   tags: ["arch:arm64"]

.gitlab/deploy_containers/deploy_containers.yml

@@ -1,32 +1,6 @@
 ---
 # deploy containers stage
-# Contains jobs which create child pipelines to deploy Agent 6 & 7 to staging repositories and to Dockerhub / GCR.
-
-#
-# Agent v6
-#
-
-deploy_containers-a6:
-  stage: deploy_containers
-  rules:
-    !reference [.on_deploy]
-  variables:
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    BUCKET_BRANCH: $BUCKET_BRANCH
-  trigger:
-    include: .gitlab/deploy_containers/deploy_containers_a6.yml
-
-deploy_containers-a6-on-failure:
-  stage: deploy_containers
-  rules:
-    !reference [.on_deploy_failure]
-  variables:
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    BUCKET_BRANCH: $BUCKET_BRANCH
-    FORCE_MANUAL: "true"
-  trigger:
-    include: .gitlab/deploy_containers/deploy_containers_a6.yml
-
+# Contains jobs which create child pipelines to deploy Agent 7 to staging repositories and to Dockerhub / GCR.
 
 deploy_containers-a7:
   stage: deploy_containers

.gitlab/package_build/include.yml

@@ -10,11 +10,8 @@
   - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
 
 include:
-  - .gitlab/package_build/deb.yml
   - .gitlab/package_build/heroku.yml
   - .gitlab/package_build/dmg.yml
   - .gitlab/package_build/installer.yml
-  - .gitlab/package_build/rpm.yml
-  - .gitlab/package_build/suse_rpm.yml
   - .gitlab/package_build/windows.yml
   - .gitlab/package_build/linux.yml

.gitlab/package_build/linux.yml

@@ -60,41 +60,35 @@
   variables:
     PACKAGE_ARCH: arm64
 
-# build Agent 6 binaries for x86_64
-datadog-agent-6-x64:
-  extends: [.agent_build_common, .agent_build_x86]
+.agent_6_build:
   variables:
     AGENT_MAJOR_VERSION: 6
     PYTHON_RUNTIMES: "2,3"
   before_script:
     - export RELEASE_VERSION=$RELEASE_VERSION_6
 
-# build Agent 7 binaries for x86_64
-datadog-agent-7-x64:
-  extends: [.agent_build_common, .agent_build_x86]
+.agent_7_build:
   variables:
     AGENT_MAJOR_VERSION: 7
     PYTHON_RUNTIMES: "3"
   before_script:
     - export RELEASE_VERSION=$RELEASE_VERSION_7
 
+# build Agent 6 binaries for x86_64
+datadog-agent-6-x64:
+  extends: [.agent_build_common, .agent_build_x86, .agent_6_build]
+
+# build Agent 7 binaries for x86_64
+datadog-agent-7-x64:
+  extends: [.agent_build_common, .agent_build_x86, .agent_7_build]
+
 # build Agent 6 binaries for arm64
 datadog-agent-6-arm64:
-  extends: [.agent_build_common, .agent_build_arm64]
-  variables:
-    AGENT_MAJOR_VERSION: 6
-    PYTHON_RUNTIMES: "2,3"
-  before_script:
-    - export RELEASE_VERSION=$RELEASE_VERSION_6
+  extends: [.agent_build_common, .agent_build_arm64, .agent_6_build]
 
 # build Agent 7 binaries for arm64
 datadog-agent-7-arm64:
-  extends: [.agent_build_common, .agent_build_arm64]
-  variables:
-    AGENT_MAJOR_VERSION: 7
-    PYTHON_RUNTIMES: "3"
-  before_script:
-    - export RELEASE_VERSION=$RELEASE_VERSION_7
+  extends: [.agent_build_common, .agent_build_arm64, .agent_7_build]
 
 .iot-agent-common:
   extends: .agent_build_common
@@ -134,3 +128,43 @@ iot-agent-armhf:
     # if we use too many compression threads or a too agressive level
     FORCED_PACKAGE_COMPRESSION_LEVEL: 5
 
+.dogstatsd_build_common:
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
+  stage: package_build
+  script:
+    - source /root/.bashrc
+    - echo "About to build for $RELEASE_VERSION"
+    - !reference [.setup_ruby_mirror_linux]
+    - !reference [.setup_python_mirror_linux]
+    - !reference [.retrieve_linux_go_deps]
+    - !reference [.cache_omnibus_ruby_deps, setup]
+    # remove artifacts from previous pipelines that may come from the cache
+    - rm -rf $OMNIBUS_PACKAGE_DIR/*
+    - inv -e omnibus.build --release-version $RELEASE_VERSION_7 --major-version 7 --python-runtimes 3 --base-dir $OMNIBUS_BASE_DIR  ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --target-project dogstatsd
+    - ls -la $OMNIBUS_PACKAGE_DIR
+    - !reference [.upload_sbom_artifacts]
+  variables:
+    KUBERNETES_CPU_REQUEST: 16
+    KUBERNETES_MEMORY_REQUEST: "32Gi"
+    KUBERNETES_MEMORY_LIMIT: "32Gi"
+  artifacts:
+    expire_in: 2 weeks
+    paths:
+      - $OMNIBUS_PACKAGE_DIR
+  cache:
+    - !reference [.cache_omnibus_ruby_deps, cache]
+
+dogstatsd-x64:
+  extends: .dogstatsd_build_common
+  needs: ["go_mod_tidy_check", "build_dogstatsd-binary_x64", "go_deps"]
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+
+dogstatsd-arm64:
+  extends: .dogstatsd_build_common
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX:$DATADOG_AGENT_ARMBUILDIMAGES
+  tags: ["arch:arm64"]
+  needs: ["go_mod_tidy_check", "build_dogstatsd-binary_arm64", "go_deps"]
+