From 4731ceaa1cd43278f243fcf5b13878a80e43c3ce Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 6 Sep 2023 10:32:44 -0400 Subject: [PATCH] [buildkite] Migrate intake pipeline (#99133) --- .buildkite/pipelines/intake.template.yml | 66 +++++++++++++++++++++++ .buildkite/pipelines/intake.yml | 67 ++++++++++++++++++++++++ build.gradle | 13 +++++ 3 files changed, 146 insertions(+) create mode 100644 .buildkite/pipelines/intake.template.yml create mode 100644 .buildkite/pipelines/intake.yml diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml new file mode 100644 index 0000000000000..8a9c153da4e0d --- /dev/null +++ b/.buildkite/pipelines/intake.template.yml @@ -0,0 +1,66 @@ +steps: + - label: sanity-check + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - wait + - label: part1 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - label: part2 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - label: part3 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - group: bwc-snapshots + steps: + - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest + timeout_in_minutes: 300 + matrix: + setup: + BWC_VERSION: $BWC_LIST + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: "{{matrix.BWC_VERSION}}" + - label: rest-compat + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - wait + - trigger: elasticsearch-dra-workflow + label: Trigger DRA snapshot workflow + async: true + build: + branch: "$BUILDKITE_BRANCH" + commit: "$BUILDKITE_COMMIT" + env: + DRA_WORKFLOW: snapshot diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml new file mode 100644 index 0000000000000..fe9f9553649c9 --- /dev/null +++ b/.buildkite/pipelines/intake.yml @@ -0,0 +1,67 @@ +# This file is auto-generated. See .buildkite/pipelines/intake.template.yml +steps: + - label: sanity-check + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - wait + - label: part1 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - label: part2 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - label: part3 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - group: bwc-snapshots + steps: + - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$$BWC_VERSION#bwcTest + timeout_in_minutes: 300 + matrix: + setup: + BWC_VERSION: ["7.17.13", "8.9.2", "8.10.0", "8.11.0"] + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: "{{matrix.BWC_VERSION}}" + - label: rest-compat + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + - wait + - trigger: elasticsearch-dra-workflow + label: Trigger DRA snapshot workflow + async: true + build: + branch: "$BUILDKITE_BRANCH" + commit: "$BUILDKITE_COMMIT" + env: + DRA_WORKFLOW: snapshot diff --git a/build.gradle b/build.gradle index cdae1701ff46c..c33489c46b53c 100644 --- a/build.gradle +++ b/build.gradle @@ -73,6 +73,14 @@ tasks.register("updateCIBwcVersions") { } } + def writeBuildkiteList = { String outputFilePath, String pipelineTemplatePath, List versions -> + def outputFile = file(outputFilePath) + def pipelineTemplate = file(pipelineTemplatePath) + + def listString = "[" + versions.collect { "\"${it}\"" }.join(", ") + "]" + outputFile.text = "# This file is auto-generated. See ${pipelineTemplatePath}\n" + pipelineTemplate.text.replaceAll('\\$BWC_LIST', listString) + } + def writeBuildkiteSteps = { String outputFilePath, String pipelineTemplatePath, String stepTemplatePath, List versions -> def outputFile = file(outputFilePath) def pipelineTemplate = file(pipelineTemplatePath) @@ -89,6 +97,11 @@ tasks.register("updateCIBwcVersions") { doLast { writeVersions(file(".ci/bwcVersions"), BuildParams.bwcVersions.allIndexCompatible) writeVersions(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible) + writeBuildkiteList( + ".buildkite/pipelines/intake.yml", + ".buildkite/pipelines/intake.template.yml", + BuildParams.bwcVersions.unreleasedIndexCompatible + ) writeBuildkiteSteps( ".buildkite/pipelines/periodic.yml", ".buildkite/pipelines/periodic.template.yml",