Skip to content

Commit

Permalink
feat: Only run required jobs for the mergequeue (#20520)
Browse files Browse the repository at this point in the history
Co-authored-by: Nicolas Schweitzer <[email protected]>
  • Loading branch information
spencergilbert and chouetz authored Nov 24, 2023
1 parent dc9e550 commit 0937077
Show file tree
Hide file tree
Showing 18 changed files with 96 additions and 2 deletions.
36 changes: 36 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,10 @@ variables:
.if_scheduled_main: &if_scheduled_main
if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "main"

# Rule to trigger jobs only when a branch matches the mergequeue pattern.
.if_mergequeue: &if_mergequeue
if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/

#
# Workflow rules
# Rules used to define whether a pipeline should run, and with which variables
Expand Down Expand Up @@ -340,26 +344,38 @@ workflow:
#

.manual:
- <<: *if_mergequeue
when: never
- when: manual
allow_failure: true

.on_a6:
- <<: *if_mergequeue
when: never
- <<: *if_version_6

.on_a6_manual:
- <<: *if_mergequeue
when: never
- <<: *if_version_6
when: manual
allow_failure: true

.on_a7:
- <<: *if_mergequeue
when: never
- <<: *if_version_7

.on_a7_manual:
- <<: *if_mergequeue
when: never
- <<: *if_version_7
when: manual
allow_failure: true

.on_dev_branch_manual:
- <<: *if_mergequeue
when: never
- <<: *if_main_branch
when: never
- <<: *if_tagged_commit
Expand All @@ -386,6 +402,8 @@ workflow:
- <<: *if_main_branch

.on_tag_or_a7:
- <<: *if_mergequeue
when: never
- <<: *if_tagged_commit
- <<: *if_version_7

Expand Down Expand Up @@ -536,6 +554,8 @@ workflow:
# rule to trigger job for internal image deployment if deploy is set or
# manually if not
.on_deploy_a7_internal_or_manual:
- <<: *if_mergequeue
when: never
- <<: *if_not_version_7
when: never
- <<: *if_deploy
Expand Down Expand Up @@ -776,6 +796,8 @@ workflow:
when: never
- <<: *if_release_branch
when: never
- <<: *if_mergequeue
when: never

.on_main_or_release_branch_or_deploy_always:
- <<: *if_deploy
Expand Down Expand Up @@ -859,6 +881,8 @@ workflow:
# In that case, the target OS versions is a subset of the
# available versions, stored in DEFAULT_KITCHEN_OSVERS
.on_default_kitchen_tests_a7:
- <<: *if_mergequeue
when: never
- <<: *if_not_version_7
when: never
- <<: *if_kitchen
Expand All @@ -867,6 +891,8 @@ workflow:
KITCHEN_OSVERS: $DEFAULT_KITCHEN_OSVERS

.on_default_new-e2e_tests_a7:
- <<: *if_mergequeue
when: never
- <<: *if_not_version_7
when: never
- <<: *if_kitchen
Expand All @@ -875,6 +901,8 @@ workflow:
E2E_OSVERS: $E2E_BRANCH_OSVERS

.on_default_kitchen_tests_a7_always:
- <<: *if_mergequeue
when: never
- <<: *if_not_version_7
when: never
- <<: *if_kitchen
Expand All @@ -894,6 +922,8 @@ workflow:
.on_security_agent_changes_or_manual:
- <<: *if_main_branch
allow_failure: true
- <<: *if_mergequeue
when: never
- changes:
paths:
- pkg/ebpf/**/*
Expand All @@ -910,6 +940,8 @@ workflow:

.on_system_probe_changes_or_manual:
- <<: *if_main_branch
- <<: *if_mergequeue
when: never
- changes:
paths:
- pkg/collector/corechecks/ebpf/**/*
Expand Down Expand Up @@ -966,3 +998,7 @@ workflow:
when: never
- <<: *if_main_branch
when: on_success

.except_mergequeue:
- <<: *if_mergequeue
when: never
2 changes: 2 additions & 0 deletions .gitlab/benchmarks/macrobenchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ variables:
tags: ["team:apm-k8s-tweaked-metal-datadog-agent", "specific:true"]
timeout: 1h
rules:
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
- when: manual
Expand Down
3 changes: 3 additions & 0 deletions .gitlab/binary_build/serverless.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
---
.build_serverless_common:
stage: binary_build
rules:
- !reference [.except_mergequeue]
- when: on_success
before_script:
- source /root/.bashrc
- !reference [.retrieve_linux_go_deps]
Expand Down
3 changes: 3 additions & 0 deletions .gitlab/binary_build/system_probe.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
---
.system-probe_build_common:
rules:
- !reference [.except_mergequeue]
- when: on_success
before_script:
- !reference [.retrieve_linux_go_deps]
# HACK: empty the build directory cache (that can come from previous runs)
Expand Down
2 changes: 2 additions & 0 deletions .gitlab/binary_build/windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# Entrypoint for the Windows Docker image
build_windows_container_entrypoint:
rules:
- !reference [.except_mergequeue]
- when: on_success
stage: binary_build
tags: ["runner:windows-docker", "windowsversion:1809"]
needs: ["lint_windows-x64", "tests_windows-x64"]
Expand Down
3 changes: 2 additions & 1 deletion .gitlab/deps_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ build_clang_arm64:

build_processed_btfhub_archive:
rules:
!reference [.on_scheduled_main_or_manual]
- !reference [.except_mergequeue]
- !reference [.on_scheduled_main_or_manual]
stage: deps_build
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES
tags: ["arch:amd64"]
Expand Down
3 changes: 3 additions & 0 deletions .gitlab/fakeintake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
docker_build_fakeintake:
stage: container_build
rules:
- !reference [.except_mergequeue]
- <<: *on_fakeintake_changes
needs: []
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3
Expand All @@ -27,6 +28,7 @@ publish_fakeintake:
extends: .docker_publish_job_definition
stage: dev_container_deploy
rules:
- !reference [.except_mergequeue]
- <<: *on_fakeintake_changes
if: $CI_COMMIT_BRANCH == "main"
- <<: *on_fakeintake_changes
Expand All @@ -45,6 +47,7 @@ publish_fakeintake_latest:
extends: .docker_publish_job_definition
stage: dev_container_deploy
rules:
- !reference [.except_mergequeue]
- <<: *on_fakeintake_changes
if: $CI_COMMIT_BRANCH == "main"
needs:
Expand Down
3 changes: 3 additions & 0 deletions .gitlab/functional_test/regression_detector.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
single-machine-performance-regression_detector:
stage: functional_test
rules:
- !reference [.except_mergequeue]
- when: on_success
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
tags: ["runner:docker"]
needs:
Expand Down
3 changes: 3 additions & 0 deletions .gitlab/functional_test/serverless.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
serverless_cold_start_performance-deb_x64:
stage: functional_test
rules:
- !reference [.except_mergequeue]
- when: on_success
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
tags: ["runner:docker"]
needs: ["go_deps", "build_serverless-deb_x64"]
Expand Down
3 changes: 3 additions & 0 deletions .gitlab/integration_test/windows.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
---
.integration_tests_windows_base:
stage: integration_test
rules:
- !reference [.except_mergequeue]
- when: on_success
needs: ["go_deps", "go_tools_deps"]
tags: ["runner:windows-docker", "windowsversion:1809"]
before_script:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
functional_test_junit_upload_security_agent:
stage: functional_test_junit_upload
rules:
- !reference [.except_mergequeue]
- when: on_success
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
tags: ["arch:amd64"]
allow_failure: true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ functional_test_junit_upload_system_probe:
# rules:
# !reference [.on_main_or_release_branch]
stage: functional_test_junit_upload
rules:
- !reference [.except_mergequeue]
- when: on_success
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
tags: ["arch:amd64"]
allow_failure: true
Expand Down
4 changes: 3 additions & 1 deletion .gitlab/kitchen_tests_upload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ kitchen_tests_upload_common:
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
tags: ["arch:amd64"]
allow_failure: true
when: always
rules:
- !reference [.except_mergequeue]
- when: always
dependencies:
- kitchen_amazonlinux_install_script_agent-a6_arm64
- kitchen_amazonlinux_install_script_agent-a6_x64
Expand Down
3 changes: 3 additions & 0 deletions .gitlab/package_deps_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@

.generate_minimized_btfs_common:
stage: package_deps_build
rules:
- !reference [.except_mergequeue]
- when: on_success
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/btf-gen$DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BTF_GEN_BUILDIMAGES
tags: ["arch:amd64"]
script:
Expand Down
6 changes: 6 additions & 0 deletions .gitlab/source_test/ebpf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ tests_ebpf_arm64:

.prepare_ebpf_functional_tests:
stage: source_test
rules:
- !reference [.except_mergequeue]
- when: on_success
needs: ["go_deps"]
artifacts:
when: always
Expand Down Expand Up @@ -103,6 +106,9 @@ prepare_ebpf_functional_tests_x64:

tests_windows_sysprobe_x64:
extends: .tests_windows_sysprobe
rules:
- !reference [.except_mergequeue]
- when: on_success
variables:
PYTHON_RUNTIMES: 3
ARCH: "x64"
3 changes: 3 additions & 0 deletions .gitlab/source_test/go_generate_check.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
---
# check that go generate has been run in the pkg/security directory
security_go_generate_check:
rules:
- !reference [.except_mergequeue]
- when: on_success
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES
tags: ["arch:amd64"]
stage: source_test
Expand Down
12 changes: 12 additions & 0 deletions .gitlab/source_test/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@ tests_flavor_iot_deb-x64:
- .rtloader_tests
- .linux_tests
- .linux_x64
rules:
- !reference [.except_mergequeue]
- when: on_success
needs: ["go_deps", "go_tools_deps"]
variables:
PYTHON_RUNTIMES: '3'
Expand All @@ -120,6 +123,9 @@ tests_flavor_dogstatsd_deb-x64:
- .rtloader_tests
- .linux_tests
- .linux_x64
rules:
- !reference [.except_mergequeue]
- when: on_success
needs: ["go_deps", "go_tools_deps"]
variables:
PYTHON_RUNTIMES: '3'
Expand All @@ -139,6 +145,9 @@ tests_flavor_heroku_deb-x64:
- .rtloader_tests
- .linux_tests
- .linux_x64
rules:
- !reference [.except_mergequeue]
- when: on_success
needs: ["go_deps", "go_tools_deps"]
variables:
PYTHON_RUNTIMES: '3'
Expand All @@ -149,6 +158,9 @@ lint_flavor_heroku_linux-x64:
extends:
- .linux_lint
- .linux_x64
rules:
- !reference [.except_mergequeue]
- when: on_success
needs: ["go_deps", "go_tools_deps"]
variables:
FLAVORS: '--flavors heroku'
Expand Down
3 changes: 3 additions & 0 deletions .gitlab/source_test/windows.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
---
.tests_windows_base:
stage: source_test
rules:
- !reference [.except_mergequeue]
- when: on_success
needs: ["go_deps", "go_tools_deps"]
tags: ["runner:windows-docker", "windowsversion:1809"]
script:
Expand Down

0 comments on commit 0937077

Please sign in to comment.