Merge branch 'main' of github.com:DataDog/datadog-agent into olivierg…

…/create-orchestrator-fwd

.github/workflows/slapr.yml

@@ -0,0 +1,28 @@
+---
+name: Slack emoji PR updates
+on:
+  pull_request_review:
+    types: [submitted]
+  pull_request:
+    types: [closed]
+
+jobs:
+  run_slapr_datadog_agent:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        channel_variables:
+          - SLACK_AGENT_PLATFORM_ID
+          - SLACK_AGENT_PLATFORM_REVIEWS_ID
+    steps:
+      - uses: DataDog/slapr@master
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          SLACK_CHANNEL_ID: "${{ secrets[matrix.channel_variables] }}"
+          SLACK_API_TOKEN: "${{ secrets.SLACK_API_TOKEN }}"
+          SLAPR_BOT_USER_ID: "${{ secrets.SLAPR_BOT_USER_ID }}"
+          SLAPR_EMOJI_REVIEW_STARTED: "review_started"
+          SLAPR_EMOJI_APPROVED: "approved2"
+          SLAPR_EMOJI_CHANGES_REQUESTED: "changes_requested"
+          SLAPR_EMOJI_MERGED: "merged"
+          SLAPR_EMOJI_CLOSED: "closed"

.gitlab-ci.yml

@@ -176,7 +176,7 @@ variables:
   # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 58b54f06414b
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 2edaabddf28b
   DATADOG_AGENT_BUILDERS: v22276738-b36b132
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
@@ -304,6 +304,10 @@ variables:
 .if_scheduled_main: &if_scheduled_main
   if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "main"
 
+# Rule to trigger jobs only when a branch matches the mergequeue pattern.
+.if_mergequeue: &if_mergequeue
+  if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
+
 #
 # Workflow rules
 # Rules used to define whether a pipeline should run, and with which variables
@@ -340,26 +344,38 @@ workflow:
 #
 
 .manual:
+  - <<: *if_mergequeue
+    when: never
   - when: manual
     allow_failure: true
 
 .on_a6:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_version_6
 
 .on_a6_manual:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_version_6
     when: manual
     allow_failure: true
 
 .on_a7:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_version_7
 
 .on_a7_manual:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_version_7
     when: manual
     allow_failure: true
 
 .on_dev_branch_manual:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_main_branch
     when: never
   - <<: *if_tagged_commit
@@ -386,6 +402,8 @@ workflow:
   - <<: *if_main_branch
 
 .on_tag_or_a7:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_tagged_commit
   - <<: *if_version_7
 
@@ -536,6 +554,8 @@ workflow:
 # rule to trigger job for internal image deployment if deploy is set or
 # manually if not
 .on_deploy_a7_internal_or_manual:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_not_version_7
     when: never
   - <<: *if_deploy
@@ -776,6 +796,8 @@ workflow:
     when: never
   - <<: *if_release_branch
     when: never
+  - <<: *if_mergequeue
+    when: never
 
 .on_main_or_release_branch_or_deploy_always:
   - <<: *if_deploy
@@ -859,6 +881,8 @@ workflow:
 # In that case, the target OS versions is a subset of the
 # available versions, stored in DEFAULT_KITCHEN_OSVERS
 .on_default_kitchen_tests_a7:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_not_version_7
     when: never
   - <<: *if_kitchen
@@ -867,6 +891,8 @@ workflow:
       KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS
 
 .on_default_new-e2e_tests_a7:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_not_version_7
     when: never
   - <<: *if_kitchen
@@ -875,6 +901,8 @@ workflow:
       E2E_OSVERS: $E2E_BRANCH_OSVERS
 
 .on_default_kitchen_tests_a7_always:
+  - <<: *if_mergequeue
+    when: never
   - <<: *if_not_version_7
     when: never
   - <<: *if_kitchen
@@ -894,6 +922,8 @@ workflow:
 .on_security_agent_changes_or_manual:
   - <<: *if_main_branch
     allow_failure: true
+  - <<: *if_mergequeue
+    when: never
   - changes:
       paths:
         - pkg/ebpf/**/*
@@ -910,6 +940,8 @@ workflow:
 
 .on_system_probe_changes_or_manual:
   - <<: *if_main_branch
+  - <<: *if_mergequeue
+    when: never
   - changes:
       paths:
         - pkg/collector/corechecks/ebpf/**/*
@@ -966,3 +998,7 @@ workflow:
     when: never
   - <<: *if_main_branch
     when: on_success
+
+.except_mergequeue:
+  - <<: *if_mergequeue
+    when: never

.gitlab/benchmarks/macrobenchmarks.yml

@@ -6,6 +6,8 @@ variables:
   tags: ["team:apm-k8s-tweaked-metal-datadog-agent", "specific:true"]
   timeout: 1h
   rules:
+    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
+      when: never
     - if: $CI_PIPELINE_SOURCE == "schedule"
       when: always
     - when: manual

.gitlab/binary_build/serverless.yml

@@ -1,6 +1,9 @@
 ---
 .build_serverless_common:
   stage: binary_build
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   before_script:
     - source /root/.bashrc
     - !reference [.retrieve_linux_go_deps]

.gitlab/binary_build/system_probe.yml

@@ -1,5 +1,8 @@
 ---
 .system-probe_build_common:
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   before_script:
     - !reference [.retrieve_linux_go_deps]
     # HACK: empty the build directory cache (that can come from previous runs)

.gitlab/binary_build/windows.yml

@@ -2,6 +2,8 @@
 # Entrypoint for the Windows Docker image
 build_windows_container_entrypoint:
   rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   stage: binary_build
   tags: ["runner:windows-docker", "windowsversion:1809"]
   needs: ["lint_windows-x64", "tests_windows-x64"]

.gitlab/deploy_dca.yml

@@ -9,7 +9,11 @@ include:
 .deploy_containers-dca-base:
   extends: .docker_publish_job_definition
   stage: deploy_dca
-  dependencies: []
+  needs:
+    - job: "docker_build_cluster_agent_amd64"
+      artifacts: false
+    - job: "docker_build_cluster_agent_arm64"
+      artifacts: false
   before_script:
     - source /root/.bashrc
     - if [[ "$VERSION" == "" ]]; then export VERSION="$(inv agent.version --major-version 7 --url-safe)"; fi
@@ -18,46 +22,34 @@ include:
     - export IMG_SOURCES="${IMG_BASE_SRC}-amd64,${IMG_BASE_SRC}-arm64"
     - export IMG_DESTINATIONS="${CLUSTER_AGENT_REPOSITORY}:${VERSION}"
 
-
 deploy_containers-dca:
   extends: .deploy_containers-dca-base
-  rules:
-    !reference [.on_deploy_a7_manual_auto_on_rc]
-
+  rules: !reference [.on_deploy_a7_manual_auto_on_rc]
 
 deploy_containers-dca-rc:
   extends: .deploy_containers-dca-base
-  rules:
-    !reference [.on_deploy_a7_rc]
+  rules: !reference [.on_deploy_a7_rc]
   variables:
     VERSION: rc
 
-
 deploy_containers-dca-latest:
   extends: .deploy_containers-dca-base
-  rules:
-    !reference [.on_deploy_a7_manual_final]
+  rules: !reference [.on_deploy_a7_manual_final]
   variables:
     VERSION: latest
 
-
 deploy_containers-dca_internal:
   extends: .deploy_containers-dca-base
-  rules:
-    !reference [.on_deploy_a7_internal_manual_final]
-
+  rules: !reference [.on_deploy_a7_internal_manual_final]
 
 deploy_containers-dca_internal-rc:
   extends: .deploy_containers-dca-base
-  rules:
-    !reference [.on_deploy_a7_internal_rc]
+  rules: !reference [.on_deploy_a7_internal_rc]
   variables:
     VERSION: rc
 
-
 deploy_containers-dca_internal-latest:
   extends: .deploy_containers-dca-base
-  rules:
-    !reference [.on_deploy_a7_internal_manual_final]
+  rules: !reference [.on_deploy_a7_internal_manual_final]
   variables:
     VERSION: latest

.gitlab/deps_build.yml

@@ -81,7 +81,8 @@ build_clang_arm64:
 
 build_processed_btfhub_archive:
   rules:
-    !reference [.on_scheduled_main_or_manual]
+    - !reference [.except_mergequeue]
+    - !reference [.on_scheduled_main_or_manual]
   stage: deps_build
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES
   tags: ["arch:amd64"]

.gitlab/fakeintake.yml

@@ -6,6 +6,7 @@
 docker_build_fakeintake:
   stage: container_build
   rules:
+    - !reference [.except_mergequeue]
     - <<: *on_fakeintake_changes
   needs: []
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3
@@ -27,6 +28,7 @@ publish_fakeintake:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
   rules:
+    - !reference [.except_mergequeue]
     - <<: *on_fakeintake_changes
       if: $CI_COMMIT_BRANCH == "main"
     - <<: *on_fakeintake_changes
@@ -45,6 +47,7 @@ publish_fakeintake_latest:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
   rules:
+    - !reference [.except_mergequeue]
     - <<: *on_fakeintake_changes
       if: $CI_COMMIT_BRANCH == "main"
   needs:

.gitlab/functional_test/regression_detector.yml

@@ -1,5 +1,8 @@
 single-machine-performance-regression_detector:
   stage: functional_test
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["runner:docker"]
   needs:

.gitlab/functional_test/security_agent.yml

@@ -25,7 +25,7 @@
     - echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/site-cookbooks/dd-security-agent-check/files/job_env.txt
     - tasks/run-test-kitchen.sh security-agent-test $AGENT_MAJOR_VERSION
     - popd
-    - inv security-agent.print-failed-tests --output-dir $DD_AGENT_TESTING_DIR/testjson
+    - inv system-probe.print-failed-tests --output-dir $DD_AGENT_TESTING_DIR/testjson
   artifacts:
     expire_in: 2 weeks
     when: always

.gitlab/functional_test/serverless.yml

@@ -1,5 +1,8 @@
 serverless_cold_start_performance-deb_x64:
   stage: functional_test
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["runner:docker"]
   needs: ["go_deps", "build_serverless-deb_x64"]

.gitlab/install_script_testing.yml

@@ -7,13 +7,10 @@ test_install_script:
     - source /root/.bashrc
     - set +x
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
-    - export APT_URL=$DEB_TESTING_S3_BUCKET
-    - export YUM_URL=$RPM_TESTING_S3_BUCKET
-    - export APT_REPO_VERSION_AGENT6="pipeline-$CI_PIPELINE_ID-a6-x86_64 6"
-    - export YUM_VERSION_PATH_AGENT6="testing/pipeline-$CI_PIPELINE_ID-a6/6"
-    - export APT_REPO_VERSION_AGENT7="pipeline-$CI_PIPELINE_ID-a7-x86_64 7"
-    - export YUM_VERSION_PATH_AGENT7="testing/pipeline-$CI_PIPELINE_ID-a7/7"
-    - inv pipeline.trigger-child-pipeline --project-name "DataDog/agent-linux-install-script" --git-ref "main" --variables "APT_URL,YUM_URL,APT_REPO_VERSION_AGENT6,APT_REPO_VERSION_AGENT7,YUM_VERSION_PATH_AGENT6,YUM_VERSION_PATH_AGENT7"
+    - export TESTING_APT_URL=$DEB_TESTING_S3_BUCKET
+    - export TESTING_YUM_URL=$RPM_TESTING_S3_BUCKET
+    - export TEST_PIPELINE_ID=$CI_PIPELINE_ID
+    - inv pipeline.trigger-child-pipeline --project-name "DataDog/agent-linux-install-script" --git-ref "main" --variables "TESTING_APT_URL,TESTING_YUM_URL,TEST_PIPELINE_ID"
   needs: ["deploy_deb_testing-a6_x64", "deploy_rpm_testing-a6_x64", "deploy_deb_testing-a7_x64", "deploy_rpm_testing-a7_x64"]
   rules:
     !reference [.on_deploy]

.gitlab/integration_test/windows.yml

@@ -1,6 +1,9 @@
 ---
 .integration_tests_windows_base:
   stage: integration_test
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   needs: ["go_deps", "go_tools_deps"]
   tags: ["runner:windows-docker", "windowsversion:1809"]
   before_script:

.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml

@@ -1,5 +1,8 @@
 functional_test_junit_upload_security_agent:
   stage: functional_test_junit_upload
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   allow_failure: true

.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml

@@ -3,6 +3,9 @@ functional_test_junit_upload_system_probe:
   # rules:
   #   !reference [.on_main_or_release_branch]
   stage: functional_test_junit_upload
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   allow_failure: true

.gitlab/kitchen_tests_upload.yml

@@ -3,7 +3,9 @@ kitchen_tests_upload_common:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   allow_failure: true
-  when: always
+  rules:
+    - !reference [.except_mergequeue]
+    - when: always
   dependencies:
     - kitchen_amazonlinux_install_script_agent-a6_arm64
     - kitchen_amazonlinux_install_script_agent-a6_x64