From 09370774595865ecb399cfac4d4e02e75cf1bd45 Mon Sep 17 00:00:00 2001 From: Spencer Gilbert Date: Fri, 24 Nov 2023 04:15:48 -0500 Subject: [PATCH] feat: Only run required jobs for the mergequeue (#20520) Co-authored-by: Nicolas Schweitzer --- .gitlab-ci.yml | 36 +++++++++++++++++++ .gitlab/benchmarks/macrobenchmarks.yml | 2 ++ .gitlab/binary_build/serverless.yml | 3 ++ .gitlab/binary_build/system_probe.yml | 3 ++ .gitlab/binary_build/windows.yml | 2 ++ .gitlab/deps_build.yml | 3 +- .gitlab/fakeintake.yml | 3 ++ .../functional_test/regression_detector.yml | 3 ++ .gitlab/functional_test/serverless.yml | 3 ++ .gitlab/integration_test/windows.yml | 3 ++ ...ional_test_junit_upload_security-agent.yml | 3 ++ ...ctional_test_junit_upload_system_probe.yml | 3 ++ .gitlab/kitchen_tests_upload.yml | 4 ++- .gitlab/package_deps_build.yml | 3 ++ .gitlab/source_test/ebpf.yml | 6 ++++ .gitlab/source_test/go_generate_check.yml | 3 ++ .gitlab/source_test/linux.yml | 12 +++++++ .gitlab/source_test/windows.yml | 3 ++ 18 files changed, 96 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 13735f5c6155e..e08bd2a21d616 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -304,6 +304,10 @@ variables: .if_scheduled_main: &if_scheduled_main if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "main" +# Rule to trigger jobs only when a branch matches the mergequeue pattern. +.if_mergequeue: &if_mergequeue + if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/ + # # Workflow rules # Rules used to define whether a pipeline should run, and with which variables @@ -340,26 +344,38 @@ workflow: # .manual: + - <<: *if_mergequeue + when: never - when: manual allow_failure: true .on_a6: + - <<: *if_mergequeue + when: never - <<: *if_version_6 .on_a6_manual: + - <<: *if_mergequeue + when: never - <<: *if_version_6 when: manual allow_failure: true .on_a7: + - <<: *if_mergequeue + when: never - <<: *if_version_7 .on_a7_manual: + - <<: *if_mergequeue + when: never - <<: *if_version_7 when: manual allow_failure: true .on_dev_branch_manual: + - <<: *if_mergequeue + when: never - <<: *if_main_branch when: never - <<: *if_tagged_commit @@ -386,6 +402,8 @@ workflow: - <<: *if_main_branch .on_tag_or_a7: + - <<: *if_mergequeue + when: never - <<: *if_tagged_commit - <<: *if_version_7 @@ -536,6 +554,8 @@ workflow: # rule to trigger job for internal image deployment if deploy is set or # manually if not .on_deploy_a7_internal_or_manual: + - <<: *if_mergequeue + when: never - <<: *if_not_version_7 when: never - <<: *if_deploy @@ -776,6 +796,8 @@ workflow: when: never - <<: *if_release_branch when: never + - <<: *if_mergequeue + when: never .on_main_or_release_branch_or_deploy_always: - <<: *if_deploy @@ -859,6 +881,8 @@ workflow: # In that case, the target OS versions is a subset of the # available versions, stored in DEFAULT_KITCHEN_OSVERS .on_default_kitchen_tests_a7: + - <<: *if_mergequeue + when: never - <<: *if_not_version_7 when: never - <<: *if_kitchen @@ -867,6 +891,8 @@ workflow: KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS .on_default_new-e2e_tests_a7: + - <<: *if_mergequeue + when: never - <<: *if_not_version_7 when: never - <<: *if_kitchen @@ -875,6 +901,8 @@ workflow: E2E_OSVERS: $E2E_BRANCH_OSVERS .on_default_kitchen_tests_a7_always: + - <<: *if_mergequeue + when: never - <<: *if_not_version_7 when: never - <<: *if_kitchen @@ -894,6 +922,8 @@ workflow: .on_security_agent_changes_or_manual: - <<: *if_main_branch allow_failure: true + - <<: *if_mergequeue + when: never - changes: paths: - pkg/ebpf/**/* @@ -910,6 +940,8 @@ workflow: .on_system_probe_changes_or_manual: - <<: *if_main_branch + - <<: *if_mergequeue + when: never - changes: paths: - pkg/collector/corechecks/ebpf/**/* @@ -966,3 +998,7 @@ workflow: when: never - <<: *if_main_branch when: on_success + +.except_mergequeue: + - <<: *if_mergequeue + when: never \ No newline at end of file diff --git a/.gitlab/benchmarks/macrobenchmarks.yml b/.gitlab/benchmarks/macrobenchmarks.yml index 10463abb4ffd3..760cda552b7cd 100644 --- a/.gitlab/benchmarks/macrobenchmarks.yml +++ b/.gitlab/benchmarks/macrobenchmarks.yml @@ -6,6 +6,8 @@ variables: tags: ["team:apm-k8s-tweaked-metal-datadog-agent", "specific:true"] timeout: 1h rules: + - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/ + when: never - if: $CI_PIPELINE_SOURCE == "schedule" when: always - when: manual diff --git a/.gitlab/binary_build/serverless.yml b/.gitlab/binary_build/serverless.yml index 61ad89c721589..8861528211fab 100644 --- a/.gitlab/binary_build/serverless.yml +++ b/.gitlab/binary_build/serverless.yml @@ -1,6 +1,9 @@ --- .build_serverless_common: stage: binary_build + rules: + - !reference [.except_mergequeue] + - when: on_success before_script: - source /root/.bashrc - !reference [.retrieve_linux_go_deps] diff --git a/.gitlab/binary_build/system_probe.yml b/.gitlab/binary_build/system_probe.yml index f63f95191011b..db27cdf1ef186 100644 --- a/.gitlab/binary_build/system_probe.yml +++ b/.gitlab/binary_build/system_probe.yml @@ -1,5 +1,8 @@ --- .system-probe_build_common: + rules: + - !reference [.except_mergequeue] + - when: on_success before_script: - !reference [.retrieve_linux_go_deps] # HACK: empty the build directory cache (that can come from previous runs) diff --git a/.gitlab/binary_build/windows.yml b/.gitlab/binary_build/windows.yml index 14beaa7fc2742..903744a359416 100644 --- a/.gitlab/binary_build/windows.yml +++ b/.gitlab/binary_build/windows.yml @@ -2,6 +2,8 @@ # Entrypoint for the Windows Docker image build_windows_container_entrypoint: rules: + - !reference [.except_mergequeue] + - when: on_success stage: binary_build tags: ["runner:windows-docker", "windowsversion:1809"] needs: ["lint_windows-x64", "tests_windows-x64"] diff --git a/.gitlab/deps_build.yml b/.gitlab/deps_build.yml index 41fd2ae486d4d..1f38d0d654f08 100644 --- a/.gitlab/deps_build.yml +++ b/.gitlab/deps_build.yml @@ -81,7 +81,8 @@ build_clang_arm64: build_processed_btfhub_archive: rules: - !reference [.on_scheduled_main_or_manual] + - !reference [.except_mergequeue] + - !reference [.on_scheduled_main_or_manual] stage: deps_build image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES tags: ["arch:amd64"] diff --git a/.gitlab/fakeintake.yml b/.gitlab/fakeintake.yml index 184877fc0fd7c..7380f52a4ffd1 100644 --- a/.gitlab/fakeintake.yml +++ b/.gitlab/fakeintake.yml @@ -6,6 +6,7 @@ docker_build_fakeintake: stage: container_build rules: + - !reference [.except_mergequeue] - <<: *on_fakeintake_changes needs: [] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3 @@ -27,6 +28,7 @@ publish_fakeintake: extends: .docker_publish_job_definition stage: dev_container_deploy rules: + - !reference [.except_mergequeue] - <<: *on_fakeintake_changes if: $CI_COMMIT_BRANCH == "main" - <<: *on_fakeintake_changes @@ -45,6 +47,7 @@ publish_fakeintake_latest: extends: .docker_publish_job_definition stage: dev_container_deploy rules: + - !reference [.except_mergequeue] - <<: *on_fakeintake_changes if: $CI_COMMIT_BRANCH == "main" needs: diff --git a/.gitlab/functional_test/regression_detector.yml b/.gitlab/functional_test/regression_detector.yml index d15eb5d88fedc..01c0e85357ed7 100644 --- a/.gitlab/functional_test/regression_detector.yml +++ b/.gitlab/functional_test/regression_detector.yml @@ -1,5 +1,8 @@ single-machine-performance-regression_detector: stage: functional_test + rules: + - !reference [.except_mergequeue] + - when: on_success image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["runner:docker"] needs: diff --git a/.gitlab/functional_test/serverless.yml b/.gitlab/functional_test/serverless.yml index 4e7425478bdfc..039a7eb029884 100644 --- a/.gitlab/functional_test/serverless.yml +++ b/.gitlab/functional_test/serverless.yml @@ -1,5 +1,8 @@ serverless_cold_start_performance-deb_x64: stage: functional_test + rules: + - !reference [.except_mergequeue] + - when: on_success image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["runner:docker"] needs: ["go_deps", "build_serverless-deb_x64"] diff --git a/.gitlab/integration_test/windows.yml b/.gitlab/integration_test/windows.yml index c2cee1b9a64e6..448247cdf36a2 100644 --- a/.gitlab/integration_test/windows.yml +++ b/.gitlab/integration_test/windows.yml @@ -1,6 +1,9 @@ --- .integration_tests_windows_base: stage: integration_test + rules: + - !reference [.except_mergequeue] + - when: on_success needs: ["go_deps", "go_tools_deps"] tags: ["runner:windows-docker", "windowsversion:1809"] before_script: diff --git a/.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml b/.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml index 5dab87253a925..7556de60f50f4 100644 --- a/.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml +++ b/.gitlab/junit_upload/functional_test_junit_upload_security-agent.yml @@ -1,5 +1,8 @@ functional_test_junit_upload_security_agent: stage: functional_test_junit_upload + rules: + - !reference [.except_mergequeue] + - when: on_success image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] allow_failure: true diff --git a/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml b/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml index 9f3f8cb8a79af..d362441aab6e5 100644 --- a/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml +++ b/.gitlab/junit_upload/functional_test_junit_upload_system_probe.yml @@ -3,6 +3,9 @@ functional_test_junit_upload_system_probe: # rules: # !reference [.on_main_or_release_branch] stage: functional_test_junit_upload + rules: + - !reference [.except_mergequeue] + - when: on_success image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] allow_failure: true diff --git a/.gitlab/kitchen_tests_upload.yml b/.gitlab/kitchen_tests_upload.yml index 7e420151156a9..140da94df94b9 100644 --- a/.gitlab/kitchen_tests_upload.yml +++ b/.gitlab/kitchen_tests_upload.yml @@ -3,7 +3,9 @@ kitchen_tests_upload_common: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] allow_failure: true - when: always + rules: + - !reference [.except_mergequeue] + - when: always dependencies: - kitchen_amazonlinux_install_script_agent-a6_arm64 - kitchen_amazonlinux_install_script_agent-a6_x64 diff --git a/.gitlab/package_deps_build.yml b/.gitlab/package_deps_build.yml index 541e979d0f34e..6bd9d38bdc1c6 100644 --- a/.gitlab/package_deps_build.yml +++ b/.gitlab/package_deps_build.yml @@ -4,6 +4,9 @@ .generate_minimized_btfs_common: stage: package_deps_build + rules: + - !reference [.except_mergequeue] + - when: on_success image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES tags: ["arch:amd64"] script: diff --git a/.gitlab/source_test/ebpf.yml b/.gitlab/source_test/ebpf.yml index c26e2de1986b6..28dd63b52364b 100644 --- a/.gitlab/source_test/ebpf.yml +++ b/.gitlab/source_test/ebpf.yml @@ -71,6 +71,9 @@ tests_ebpf_arm64: .prepare_ebpf_functional_tests: stage: source_test + rules: + - !reference [.except_mergequeue] + - when: on_success needs: ["go_deps"] artifacts: when: always @@ -103,6 +106,9 @@ prepare_ebpf_functional_tests_x64: tests_windows_sysprobe_x64: extends: .tests_windows_sysprobe + rules: + - !reference [.except_mergequeue] + - when: on_success variables: PYTHON_RUNTIMES: 3 ARCH: "x64" diff --git a/.gitlab/source_test/go_generate_check.yml b/.gitlab/source_test/go_generate_check.yml index 2a64c41e4136b..c9b596cc39e5c 100644 --- a/.gitlab/source_test/go_generate_check.yml +++ b/.gitlab/source_test/go_generate_check.yml @@ -1,6 +1,9 @@ --- # check that go generate has been run in the pkg/security directory security_go_generate_check: + rules: + - !reference [.except_mergequeue] + - when: on_success image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES tags: ["arch:amd64"] stage: source_test diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 92f4fd7145060..a9f1db150506d 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -101,6 +101,9 @@ tests_flavor_iot_deb-x64: - .rtloader_tests - .linux_tests - .linux_x64 + rules: + - !reference [.except_mergequeue] + - when: on_success needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -120,6 +123,9 @@ tests_flavor_dogstatsd_deb-x64: - .rtloader_tests - .linux_tests - .linux_x64 + rules: + - !reference [.except_mergequeue] + - when: on_success needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -139,6 +145,9 @@ tests_flavor_heroku_deb-x64: - .rtloader_tests - .linux_tests - .linux_x64 + rules: + - !reference [.except_mergequeue] + - when: on_success needs: ["go_deps", "go_tools_deps"] variables: PYTHON_RUNTIMES: '3' @@ -149,6 +158,9 @@ lint_flavor_heroku_linux-x64: extends: - .linux_lint - .linux_x64 + rules: + - !reference [.except_mergequeue] + - when: on_success needs: ["go_deps", "go_tools_deps"] variables: FLAVORS: '--flavors heroku' diff --git a/.gitlab/source_test/windows.yml b/.gitlab/source_test/windows.yml index 31ec9d153e6f0..c12ec331aa718 100644 --- a/.gitlab/source_test/windows.yml +++ b/.gitlab/source_test/windows.yml @@ -1,6 +1,9 @@ --- .tests_windows_base: stage: source_test + rules: + - !reference [.except_mergequeue] + - when: on_success needs: ["go_deps", "go_tools_deps"] tags: ["runner:windows-docker", "windowsversion:1809"] script: