Merge branch 'main' into chouquette/omnibus_cache

.circleci/config.yml

@@ -15,7 +15,7 @@ experimental:
 templates:
   job_template: &job_template
     docker:
-      - image: gcr.io/datadoghq/agent-circleci-runner:v31802788-2dee8fe9
+      - image: gcr.io/datadoghq/agent-circleci-runner:v31988376-bfbb3afb
         environment:
           USE_SYSTEM_LIBS: "1"
     working_directory: /go/src/github.com/DataDog/datadog-agent
@@ -119,18 +119,6 @@ jobs:
           name: run integration tests
           command: inv -e integration-tests --race --remote-docker
 
-  release_note:
-    <<: *job_template
-    steps:
-      - restore_cache: *restore_source
-      - restore_cache: *restore_deps
-      - run:
-          command: pip install -r tasks/libs/requirements-github.txt
-          name: Install additional requirements
-      - run:
-          command: inv -e lint-releasenote
-          name: run PR check for release note
-
   go_linting:
     <<: *job_template
     resource_class: xlarge
@@ -187,20 +175,6 @@ jobs:
           #SC2059: Don't use variables in the printf format string. Use printf "..%s.." "$foo".
           command: shellcheck --severity=info -e SC2059 -e SC2028 --shell=bash ./cmd/**/*.sh ./omnibus/package-scripts/*/*
 
-  python_linting:
-    <<: *job_template
-    steps:
-      - restore_cache: *restore_source
-      - restore_cache: *restore_deps
-      - run:
-          name: setting env vars for click
-          command: |
-            echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV
-            echo 'export LANG="C.UTF-8"' >> $BASH_ENV
-      - run:
-          name: lint python files
-          command: inv -e linter.python
-
   component_linting:
     <<: *job_template
     steps:
@@ -280,13 +254,6 @@ workflows:
       - integration_tests:
           requires:
             - dependencies
-      - release_note:
-          filters:
-            branches:
-              ignore:
-                - main
-          requires:
-            - dependencies
       - go_linting:
           requires:
             - dependencies
@@ -302,9 +269,6 @@ workflows:
       - shell_linting:
           requires:
             - dependencies
-      - python_linting:
-          requires:
-            - dependencies
       - component_linting:
           requires:
             - dependencies

.copyright-overrides.yml

@@ -20,6 +20,7 @@ github.com/GoogleCloudPlatform/docker-credential-gcr: Copyright 2016 Google, Inc
 github.com/aws/aws-lambda-go: Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 github.com/aws/aws-sdk-go: ["Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.", "Copyright 2014-2015 Stripe, Inc."]
 github.com/aws/aws-sdk-go-v2: ["Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.", "Copyright 2014-2015 Stripe, Inc."]
+github.com/awslabs/amazon-ecr-credential-helper/*: "Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved."
 github.com/containerd/*: Copyright 2012-2015 Docker, Inc.
 github.com/coreos/*: Copyright 2017 CoreOS, Inc
 github.com/docker/*: Copyright 2012-2017 Docker, Inc.

.ddqa/config.toml

@@ -121,7 +121,7 @@ github_labels = ["team/remote-config"]
 jira_project = "FA"
 jira_issue_type = "QA"
 jira_statuses = ["To Do", "In Progress", "Done"]
-github_team = "fleet-automation"
+github_team = "Fleet"
 github_labels = ["team/fleet-automation"]
 
 [teams."Container Integrations"]

.github/CODEOWNERS

@@ -257,6 +257,7 @@
 /comp/systray @DataDog/windows-agent
 /comp/trace @DataDog/agent-apm
 /comp/updater @DataDog/fleet
+/comp/agent/cloudfoundrycontainer @DataDog/platform-integrations
 /comp/agent/jmxlogger @DataDog/agent-metrics-logs
 /comp/aggregator/diagnosesendermanager @DataDog/agent-shared-components
 /comp/checks/agentcrashdetect @DataDog/windows-kernel-integrations

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
           go-version-file: ".go-version"
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v3.24.10
         with:
           languages: ${{ matrix.language }}
           setup-python-dependencies: false
@@ -62,4 +62,4 @@ jobs:
           invoke agent.build --build-exclude=systemd
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3
+        uses: github/codeql-action/analyze@v3.24.10

.github/workflows/cws-btfhub-sync.yml

@@ -2,6 +2,17 @@ name: "CWS BTFHub constants sync"
 
 on:
   workflow_dispatch:
+    inputs:
+      base_branch:
+        description: 'Base branch to target'
+        required: false
+        default: 'main'
+        type: string
+      force_refresh:
+        description: 'Force refresh of the constants'
+        required: false
+        default: 'false'
+        type: boolean
   schedule:
     - cron: '30 4 * * 5' # at 4:30 UTC on Friday
 
@@ -16,6 +27,8 @@ jobs:
 
       - name: Checkout datadog-agent repository
         uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.base_branch || 'main' }}
 
       - name: Checkout btfhub-archive repository
         uses: actions/checkout@v4
@@ -47,7 +60,7 @@ jobs:
 
       - name: Sync constants
         run: |
-          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive
+          inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive ${{ inputs.force_refresh && '--force-refresh' || '' }}
 
       - uses: stefanzweifel/git-auto-commit-action@v5
         id: commit-creator
@@ -69,7 +82,7 @@ jobs:
               owner,
               repo,
               head: '${{ steps.branch-name.outputs.BRANCH_NAME }}',
-              base: 'main',
+              base: '${{ inputs.base_branch || 'main' }}',
               body: [
                 '### What does this PR do?',
                 'This PR syncs the BTFHub constants used by CWS',

.github/workflows/label-analysis.yml

@@ -20,10 +20,37 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Install Python dependencies
-        run: pip install -r tasks/requirements.txt
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: pip install -r requirements.txt -r tasks/requirements.txt
       - name: Auto assign team label
         run: inv -e github.assign-team-label --pr-id='${{ github.event.pull_request.number }}'
+  release-note-check:
+    if: github.triggering_actor != 'dd-devflow[bot]'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: '**/requirements*.txt'
+      - name: Install dependencies
+        run: pip install -r requirements.txt -r tasks/requirements.txt
+      - name: Check release note
+        env:
+          BRANCH_NAME: ${{ github.head_ref }}
+          PR_ID: ${{ github.event.pull_request.number }}
+        run: inv -e linter.releasenote
   fetch-labels:
     needs: assign-team-label
     if: github.triggering_actor != 'dd-devflow[bot]'
@@ -37,7 +64,7 @@ jobs:
           labels="$(gh pr view '${{ github.event.pull_request.number }}' --json labels --jq '[.labels[].name] | (join(" "))')"
           echo "Fetched labels for PR ${{github.event.number}}: $labels"
           echo "LABELS=$labels" >> "$GITHUB_OUTPUT"
-  team-label:
+  team-label-check:
     needs: fetch-labels
     if: github.triggering_actor != 'dd-devflow[bot]'
     runs-on: ubuntu-latest
@@ -58,7 +85,7 @@ jobs:
           exit 1
         env:
           LABELS: ${{ needs.fetch-labels.outputs.LABELS}}
-  skip-qa:
+  skip-qa-check:
     needs: fetch-labels
     if: github.triggering_actor != 'dd-devflow[bot]'
     runs-on: ubuntu-latest

.github/workflows/serverless-binary-size.yml

@@ -77,7 +77,7 @@ jobs:
       ### Steps below only run if size diff > SIZE_ALLOWANCE ###
 
       - name: Install graphviz
-        uses: ts-graphviz/setup-graphviz@v1
+        uses: ts-graphviz/setup-graphviz@v2
         if: steps.compare.outputs.diff > env.SIZE_ALLOWANCE
 
       - name: Install digraph

.gitlab-ci.yml

@@ -77,7 +77,6 @@ stages:
   - pkg_metrics
   - packaging
   - kitchen_deploy
-  - e2e_pre_test
   - kitchen_testing
   - container_build
   - container_scan
@@ -92,6 +91,7 @@ stages:
   - choco_deploy
   - internal_image_deploy
   - install_script_testing
+  - e2e_pre_test
   - e2e
   - kitchen_cleanup
   - functional_test
@@ -154,7 +154,7 @@ variables:
   INTEGRATION_WHEELS_CACHE_BUCKET: dd-agent-omnibus
   S3_DD_AGENT_OMNIBUS_LLVM_URI: s3://dd-agent-omnibus/llvm
   S3_DD_AGENT_OMNIBUS_BTFS_URI: s3://dd-agent-omnibus/btfs
-  BTFHUB_ARCHIVE_BRANCH: no-kmod
+  BTFHUB_ARCHIVE_BRANCH: main
   GENERAL_ARTIFACTS_CACHE_BUCKET_URL: https://dd-agent-omnibus.s3.amazonaws.com
   S3_DSD6_URI: s3://dsd6-staging
   RELEASE_VERSION_6: nightly
@@ -164,19 +164,15 @@ variables:
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to _test_only
   DATADOG_AGENT_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BUILDIMAGES: v31802788-2dee8fe9
+  DATADOG_AGENT_BUILDIMAGES: v31988376-bfbb3afb
   DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_WINBUILDIMAGES: v31802788-2dee8fe9
+  DATADOG_AGENT_WINBUILDIMAGES: v31988376-bfbb3afb
   DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_ARMBUILDIMAGES: v31802788-2dee8fe9
+  DATADOG_AGENT_ARMBUILDIMAGES: v31988376-bfbb3afb
   DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v31802788-2dee8fe9
-  DATADOG_AGENT_KERNEL_MATRIX_TESTING_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_KERNEL_MATRIX_TESTING_BUILDIMAGES: v31802788-2dee8fe9
-  DATADOG_AGENT_NIKOS_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_NIKOS_BUILDIMAGES: v31802788-2dee8fe9
+  DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v31988376-bfbb3afb
   DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: ""
-  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v31802788-2dee8fe9
+  DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v31988376-bfbb3afb
   DATADOG_AGENT_BUILDERS: v28719426-b6a4fd9
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
@@ -228,6 +224,9 @@ variables:
   MACOS_GITHUB_APP_ID_SSM_NAME: ci.datadog-agent.macos_github_app_id  # agent-ci-experience
   MACOS_GITHUB_INSTALLATION_ID_SSM_NAME: ci.datadog-agent.macos_github_installation_id  # agent-ci-experience
   MACOS_GITHUB_KEY_SSM_NAME: ci.datadog-agent.macos_github_key_b64  # agent-ci-experience
+  MACOS_GITHUB_APP_ID_2_SSM_NAME: ci.datadog-agent.macos_github_app_id_2  # agent-ci-experience
+  MACOS_GITHUB_INSTALLATION_ID_2_SSM_NAME: ci.datadog-agent.macos_github_installation_id_2  # agent-ci-experience
+  MACOS_GITHUB_KEY_2_SSM_NAME: ci.datadog-agent.macos_github_key_b64_2  # agent-ci-experience
   RPM_GPG_KEY_SSM_NAME: ci.datadog-agent.rpm_signing_private_key_v2_${RPM_GPG_KEY_ID}  # agent-build-and-releases
   RPM_SIGNING_PASSPHRASE_SSM_NAME: ci.datadog-agent.rpm_signing_key_passphrase_v2_${RPM_GPG_KEY_ID}  # agent-build-and-releases
   SMP_ACCOUNT_ID_SSM_NAME: ci.datadog-agent.single-machine-performance-account-id  # single-machine-performance
@@ -360,6 +359,15 @@ variables:
 .if_mergequeue: &if_mergequeue
   if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
 
+.fakeintake_changes: &fakeintake_changes
+  changes:
+    paths:
+      - "test/fakeintake/**/*"
+      - .gitlab/binary_build/fakeintake.yml
+      - .gitlab/container_build/fakeintake.yml
+      - .gitlab/dev_container_deploy/fakeintake.yml
+    compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+
 #
 # Workflow rules
 # Rules used to define whether a pipeline should run, and with which variables
@@ -765,6 +773,23 @@ workflow:
 
 # New E2E related rules
 
+.on_e2e_or_fakeintake_changes_or_manual:
+  - <<: *if_main_branch
+  - !reference [.except_mergequeue]
+  - <<: *fakeintake_changes
+    variables:
+      FAKEINTAKE_IMAGE_OVERRIDE: "public.ecr.aws/datadog/fakeintake:v$CI_COMMIT_SHORT_SHA"
+    when: on_success
+  - changes:
+      paths:
+        - test/new-e2e/pkg/**/*
+        - test/new-e2e/test-infra-definition/*
+        - test/new-e2e/go.mod
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+    when: on_success
+  - when: manual
+    allow_failure: true
+
 
 .on_e2e_main_release_or_rc: # This rule is used as a base for all new-e2e rules
   - <<: *if_disable_e2e_tests
@@ -919,7 +944,7 @@ workflow:
   - when: manual
     allow_failure: true
 
-.on_updater_or_e2e_changes_or_manual:
+.on_installer_or_e2e_changes_or_manual:
   - <<: *if_disable_e2e_tests
     when: never
   - !reference [.except_mergequeue]
@@ -933,10 +958,10 @@ workflow:
       paths:
         - .gitlab/**/*
         - omnibus/config/**/*
-        - pkg/updater/**/*
-        - cmd/updater/**/*
-        - test/new-e2e/tests/updater/**/*
-        - tasks/updater.py
+        - pkg/installer/**/*
+        - cmd/installer/**/*
+        - test/new-e2e/tests/installer/**/*
+        - tasks/installer.py
         - test/new-e2e/go.mod
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
     when: on_success
@@ -995,23 +1020,12 @@ workflow:
         - .go-version
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
-.on_fakeintake_changes: &on_fakeintake_changes
-  changes:
-    - "test/fakeintake/**/*"
-    - .gitlab/binary_build/fakeintake.yml
-    - .gitlab/container_build/fakeintake.yml
-    - .gitlab/dev_container_deploy/fakeintake.yml
-
-.on_fakeintake_changes_on_main_or_manual:
-  - <<: *on_fakeintake_changes
-    if: $CI_COMMIT_BRANCH == "main"
-  - <<: *on_fakeintake_changes
-    when: manual
-    allow_failure: true
+.on_fakeintake_changes:
+  - <<: *fakeintake_changes
 
 .on_fakeintake_changes_on_main:
-  - <<: *on_fakeintake_changes
-    if: $CI_COMMIT_BRANCH == "main"
+  - <<: *fakeintake_changes
+    <<: *if_main_branch
 
 .fast_on_dev_branch_only:
   - <<: *if_main_branch
@@ -1039,4 +1053,17 @@ workflow:
         - .gitlab-ci.yml
         - .gitlab/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
-
+
+.on_invoke_tasks_changes:
+  - <<: *if_main_branch
+  - changes:
+      paths:
+        - tasks/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+
+.on_kitchen_invoke_tasks_changes:
+  - <<: *if_main_branch
+  - changes:
+      paths:
+        - test/kitchen/tasks/**/*
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916