Merge branch 'main' into dependabot/github_actions/actions/checkout-4…

….2.2

.github/workflows/create_rc_pr.yml

@@ -5,10 +5,12 @@ on:
   schedule:
     - cron: '0 14 * * 1,3,5' # Run on Monday, Wednesday, and Friday at 14:00 UTC
     - cron: '0 8 * * 1,3,5' # Same as above but at 08:00 UTC, to warn agent-integrations team about releasing
+    - cron: '0 9 * * 1' # Run Agent 6 workflow on Monday at 09:00 UTC
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
+  AGENT6_RELEASE_BRANCH: '6.53.x'
+  IS_AGENT6_RELEASE: ${{ github.event.schedule == '0 9 * * 1' }}
 permissions: {}
 
 jobs:
@@ -19,18 +21,21 @@ jobs:
           warning: ${{ steps.warning.outputs.value }}
         steps:
             - name: Checkout repository
-              uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+              uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
               with:
                 sparse-checkout: 'tasks'
                 persist-credentials: false
 
             - name: Install python
+              if: ${{ env.IS_AGENT6_RELEASE == 'false' }}
               uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
               with:
                 python-version: 3.11
                 cache: "pip"
 
             - name: Install Python dependencies
+              if: ${{ env.IS_AGENT6_RELEASE == 'false' }}
               run: |
                 python -m pip install --upgrade pip
                 pip install -r requirements.txt
@@ -40,7 +45,11 @@ jobs:
             - name: Determine the release active branches
               id: branches
               run: |
-                echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT
+                if ${{ env.IS_AGENT6_RELEASE == 'true' }}; then
+                  echo "value=[\"$AGENT6_RELEASE_BRANCH\"]" >> $GITHUB_OUTPUT
+                else
+                  echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT
+                fi
 
             - name: Set the warning option
               id: warning
@@ -93,11 +102,12 @@ jobs:
                 fi
 
             - name: Create RC PR
-              if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}}
+              if: ${{ steps.check_for_changes.outputs.CHANGES == 'true' || env.IS_AGENT6_RELEASE == 'true' }}
               env:
                 MATRIX: ${{ matrix.value }}
               run: |
-                git config user.name "github-actions[bot]"
-                git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-                git fetch
-                inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }}
+                if ${{ env.IS_AGENT6_RELEASE == 'true' }}; then
+                  inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }} --patch-version
+                else
+                  inv -e release.create-rc -r "$MATRIX" --slack-webhook=${{ secrets.AGENT_RELEASE_SYNC_SLACK_WEBHOOK }}
+                fi

.github/workflows/serverless-binary-size.yml

@@ -43,7 +43,7 @@ jobs:
           persist-credentials: false
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
 
       - name: Previous binary size and dependencies
         id: previous

.github/workflows/serverless-integration.yml

@@ -54,7 +54,7 @@ jobs:
           platforms: amd64,arm64
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
+        uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1
 
       - name: Create raw logs directory
         id: rawlogs

.gitlab-ci.yml

@@ -584,8 +584,10 @@ workflow:
     when: always
 
 .on_deploy_stable_or_beta_repo_branch:
+  - !reference [.except_mergequeue]
   - <<: *if_not_stable_or_beta_repo_branch
-    when: never
+    when: manual
+    allow_failure: true
   - <<: *if_deploy
 
 .on_deploy_stable_or_beta_repo_branch_manual:
@@ -614,7 +616,8 @@ workflow:
 
 .except_no_tests_no_deploy:
   - if: $DEPLOY_AGENT == "false" && $DDR_WORKFLOW_ID == null && $RUN_E2E_TESTS == "off"
-    when: never
+    when: manual
+    allow_failure: true
 
 .on_main_or_release_branch:
   - <<: *if_main_branch

.gitlab/choco_build/choco_build.yml

@@ -14,8 +14,13 @@ windows_choco_offline_7_x64:
   script:
     - $ErrorActionPreference = "Stop"
     - Get-ChildItem omnibus\pkg
-    - copy omnibus\pkg\*.msi .\chocolatey\tools-offline\
-    - docker run --rm -v "$(Get-Location):c:\mnt" registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\chocopack.bat offline
+    - copy omnibus\pkg\*.msi .\chocolatey\datadog-agent\offline\tools\
+    - >
+      docker run --rm
+      -v "$(Get-Location):c:\mnt"
+      -e AWS_NETWORKING=true
+      registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
+      powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -InstallMethod offline -Flavor $FLAVOR -InstallDeps 1"
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
     - copy build-out\*.nupkg omnibus\pkg
   artifacts:
@@ -24,12 +29,11 @@ windows_choco_offline_7_x64:
       - omnibus/pkg
 
 # The online version of the choco job gets the msi package through the gitlab artifacts
-windows_choco_online_7_x64:
+.windows_choco_online_7_x64:
   rules:
     !reference [.on_deploy_stable_or_beta_repo_branch]
   stage: choco_and_install_script_build
   tags: ["runner:windows-docker", "windowsversion:1809"]
-  needs: ["deploy_packages_windows-x64-7"]
   variables:
     ARCH: "x64"
   script:
@@ -43,10 +47,12 @@ windows_choco_online_7_x64:
     - >
       docker run --rm
       -v "$(Get-Location):c:\mnt"
+      -e CI_PROJECT_NAME=${CI_PROJECT_NAME}
       -e CI_PIPELINE_ID=${CI_PIPELINE_ID}
       -e BUCKET_BRANCH="$BUCKET_BRANCH"
+      -e AWS_NETWORKING=true
       registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
-      c:\mnt\tasks\winbuildscripts\chocopack.bat online c:\mnt\temp
+      powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -InstallMethod online -MSIDirectory c:\mnt\temp -Flavor $FLAVOR -InstallDeps 1"
     - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" }
     - Remove-Item -Path "temp\" -Recurse -Force
     - copy build-out\*.nupkg omnibus\pkg
@@ -58,3 +64,23 @@ windows_choco_online_7_x64:
       - omnibus/pkg
   # Sometimes Chocolatey is flakey
   retry: 2
+
+windows_choco_online_7_x64:
+  extends: .windows_choco_online_7_x64
+  # On dev/PR branches:
+  #   - if the job is run manually it will create a package, but before the
+  #     package can be installed, the deploy_windows_testing-a7 job must
+  #     be run to push the MSI to the dd-agent-mstesting bucket.
+  needs: ["windows_msi_and_bosh_zip_x64-a7"]
+  variables:
+    FLAVOR: "datadog-agent"
+
+windows_choco_online_7_x64-fips:
+  extends: .windows_choco_online_7_x64
+  # On dev/PR branches:
+  #   - if the job is run manually it will create a package, but before the
+  #     package can be installed, the deploy_windows_testing-a7-fips job must
+  #     be run to push the MSI to the dd-agent-mstesting bucket.
+  needs: ["windows_msi_and_bosh_zip_x64-a7-fips"]
+  variables:
+    FLAVOR: "datadog-fips-agent"

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 9c7c5005ca28
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 4b4112f5f64d

.gitlab/deploy_packages/e2e.yml

@@ -1,31 +1,5 @@
 # Jobs that deploy agent packages on QA environment, to be used by e2e tests
 
-qa_agent_oci:
-  extends: .docker_publish_job_definition
-  stage: deploy_packages
-  rules:
-    - !reference [.on_installer_or_e2e_changes]
-    - !reference [.manual]
-  needs:
-    - deploy_agent_oci
-  variables:
-    IMG_REGISTRIES: agent-qa
-    IMG_SOURCES: registry.ddbuild.io/ci/remote-updates/datadog-agent:pipeline-${CI_PIPELINE_ID}
-    IMG_DESTINATIONS: agent-package:pipeline-${CI_PIPELINE_ID}
-
-qa_installer_oci:
-  extends: .docker_publish_job_definition
-  stage: deploy_packages
-  rules:
-    - !reference [.on_installer_or_e2e_changes]
-    - !reference [.manual]
-  needs:
-    - deploy_installer_oci
-  variables:
-    IMG_REGISTRIES: agent-qa
-    IMG_SOURCES: registry.ddbuild.io/ci/remote-updates/datadog-installer:pipeline-${CI_PIPELINE_ID}
-    IMG_DESTINATIONS: installer-package:pipeline-${CI_PIPELINE_ID}
-
 qa_installer_script:
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   stage: deploy_packages

.gitlab/deploy_packages/oci.yml

@@ -24,9 +24,10 @@ include:
     - datadog-package push registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:${VERSION} ${OMNIBUS_PACKAGE_DIR}/${OCI_PRODUCT}-${VERSION}.oci.tar
     # This is used for E2E tests. Doesn't cost more than an additional tag to the registry.
     - datadog-package push registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} ${OMNIBUS_PACKAGE_DIR}/${OCI_PRODUCT}-${VERSION}.oci.tar
-    # Used for install scripts e2e tests
+    # Used for e2e tests
     - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} ${VERSION}
     - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} ${CI_COMMIT_SHA}
+    - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} pipeline-${CI_PIPELINE_ID}
   variables:
     MAJOR_VERSION: 7
 

.gitlab/e2e/e2e.yml

@@ -412,8 +412,8 @@ new-e2e-installer:
     - deploy_rpm_testing-a7_x64
     - deploy_suse_rpm_testing_arm64-a7
     - deploy_suse_rpm_testing_x64-a7
-    - qa_installer_oci
-    - qa_agent_oci
+    - deploy_installer_oci
+    - deploy_agent_oci
   variables:
     TARGETS: ./tests/installer/unix
     TEAM: fleet
@@ -428,8 +428,8 @@ new-e2e-installer-windows:
   needs:
     - !reference [.needs_new_e2e_template]
     - deploy_windows_testing-a7
-    - qa_installer_oci
-    - qa_agent_oci
+    - deploy_installer_oci
+    - deploy_agent_oci
   before_script:
     # CURRENT_AGENT_VERSION is used to verify the installed agent version
     # Must run before new_e2e_template changes the aws profile
@@ -471,8 +471,8 @@ new-e2e-installer-ansible:
     - deploy_rpm_testing-a7_x64
     - deploy_suse_rpm_testing_arm64-a7
     - deploy_suse_rpm_testing_x64-a7
-    - qa_installer_oci
-    - qa_agent_oci
+    - deploy_installer_oci
+    - deploy_agent_oci
   variables:
     TARGETS: ./tests/installer/unix
     TEAM: fleet
@@ -663,8 +663,8 @@ generate-flakes-finder-pipeline:
     - deploy_suse_rpm_testing_arm64-a7
     - deploy_suse_rpm_testing_x64-a7
     - deploy_windows_testing-a7
-    - qa_installer_oci
-    - qa_agent_oci
+    - deploy_installer_oci
+    - deploy_agent_oci
     - qa_cws_instrumentation
     - qa_dca
     - qa_dogstatsd

.gitlab/notify/notify.yml

@@ -28,8 +28,23 @@ notify:
     - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN
     - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY
     - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt
-    - invoke -e notify.send-message
-    - invoke -e notify.check-consistent-failures
+    - |
+      # Do not send notifications if this is a child pipeline of another repo
+      # The triggering repo should already have its own notification system
+      if [ "$CI_PIPELINE_SOURCE" != "pipeline" ]; then
+        if [ "$DEPLOY_AGENT" = "true" ]; then
+          invoke -e notify.send-message --notification-type "deploy"
+        elif [ "$CI_PIPELINE_SOURCE" != "push" ]; then
+          invoke -e notify.send-message --notification-type "trigger"
+        else
+          invoke -e notify.send-message --notification-type "merge"
+        fi
+        if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
+          invoke notify.check-consistent-failures
+        fi
+      else
+        echo "This pipeline was triggered by another repository, skipping notification."
+      fi
 
 send_pipeline_stats:
   stage: notify
@@ -65,8 +80,8 @@ notify_github:
   script:
     - !reference [.setup_agent_github_app]
     # Python 3.12 changes default behavior how packages are installed.
-    # In particular, --break-system-packages command line option is 
-    # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615      
+    # In particular, --break-system-packages command line option is
+    # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615
     - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages
     - messagefile="$(mktemp)"
     - echo "Use this command from [test-infra-definitions](https://github.com/DataDog/test-infra-definitions) to manually test this PR changes on a VM:" >> "$messagefile"
@@ -83,15 +98,16 @@ notify_gitlab_ci_changes:
   needs: [compute_gitlab_ci_config]
   tags: ["arch:amd64"]
   rules:
+    - !reference  [.except_mergequeue]
     - changes:
         paths:
           - .gitlab-ci.yml
           - .gitlab/**/*.yml
         compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
   script:
     # Python 3.12 changes default behavior how packages are installed.
-    # In particular, --break-system-packages command line option is 
-    # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615    
+    # In particular, --break-system-packages command line option is
+    # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615
     - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages
     - !reference [.setup_agent_github_app]
     - inv -e notify.gitlab-ci-diff --from-diff artifacts/diff.gitlab-ci.yml --pr-comment

.gitlab/package_build/windows.yml

@@ -36,7 +36,6 @@
       -e BUNDLE_MIRROR__RUBYGEMS__ORG=${BUNDLE_MIRROR__RUBYGEMS__ORG}
       -e PIP_INDEX_URL=${PIP_INDEX_URL}
       -e API_KEY_ORG2=${API_KEY_ORG2}
-      -e OMNIBUS_GIT_CACHE_DIR=${Env:TEMP}/${CI_PIPELINE_ID}/omnibus-git-cache
       -e AGENT_FLAVOR=${AGENT_FLAVOR}
       registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES}
       powershell -C "c:\mnt\tasks\winbuildscripts\Build-AgentPackages.ps1 -BuildOutOfSource 1 -InstallDeps 1 -CheckGoVersion 1"

CHANGELOG-DCA.rst

@@ -2,6 +2,33 @@
 Release Notes
 =============
 
+.. _Release Notes_7.60.0:
+
+7.60.0
+======
+
+.. _Release Notes_7.60.0_Prelude:
+
+Prelude
+-------
+
+Released on: 2024-12-16
+Pinned to datadog-agent v7.60.0: `CHANGELOG <https://github.com/DataDog/datadog-agent/blob/main/CHANGELOG.rst#7600>`_.
+
+
+.. _Release Notes_7.60.0_Bug Fixes:
+
+Bug Fixes
+---------
+
+- Fixes bug where incorrect timestamp would be used for unbundled Kubernetes events.
+
+- Fixed an issue in the KSM check when it's configured with the option
+  ``pod_collection_mode`` set to ``node_kubelet``. Previously, the check could
+  fail to start if there was a timeout while contacting the API server. This
+  issue has now been resolved.
+
+
 .. _Release Notes_7.59.1:
 
 7.59.1