diff --git a/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap b/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap index 39cd3fe07beb4..6df8ca8b63438 100644 --- a/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap +++ b/.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap @@ -3,47 +3,53 @@ exports[`generatePipelines should generate correct pipelines with a non-docs change 1`] = ` [ { - "steps": [ - { - "group": "bwc-snapshots", - "steps": [ - { - "agents": { - "buildDirectory": "/dev/shm/bk", - "image": "family/elasticsearch-ubuntu-2004", - "machineType": "custom-32-98304", - "provider": "gcp", - }, - "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest", - "env": { - "BWC_VERSION": "{{matrix.BWC_VERSION}}", - }, - "label": "{{matrix.BWC_VERSION}} / bwc-snapshots", - "matrix": { - "setup": { - "BWC_VERSION": [ - "7.17.14", - "8.10.3", - "8.11.0", - ], + "name": "bwc-snapshots", + "pipeline": { + "steps": [ + { + "group": "bwc-snapshots", + "steps": [ + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest", + "env": { + "BWC_VERSION": "{{matrix.BWC_VERSION}}", }, + "label": "{{matrix.BWC_VERSION}} / bwc-snapshots", + "matrix": { + "setup": { + "BWC_VERSION": [ + "7.17.14", + "8.10.3", + "8.11.0", + ], + }, + }, + "timeout_in_minutes": 300, }, - "timeout_in_minutes": 300, - }, - ], - }, - ], + ], + }, + ], + }, }, { - "env": { - "CUSTOM_ENV_VAR": "value", - }, - "steps": [ - { - "command": "echo 'hello world'", - "label": "test-step", + "name": "using-defaults", + "pipeline": { + "env": { + "CUSTOM_ENV_VAR": "value", }, - ], + "steps": [ + { + "command": "echo 'hello world'", + "label": "test-step", + }, + ], + }, }, ] `; @@ -51,19 +57,22 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha exports[`generatePipelines should generate correct pipelines with only docs changes 1`] = ` [ { - "steps": [ - { - "agents": { - "buildDirectory": "/dev/shm/bk", - "image": "family/elasticsearch-ubuntu-2004", - "machineType": "custom-32-98304", - "provider": "gcp", + "name": "docs-check", + "pipeline": { + "steps": [ + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check", + "label": "docs-check", + "timeout_in_minutes": 300, }, - "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed precommit :docs:check", - "label": "docs-check", - "timeout_in_minutes": 300, - }, - ], + ], + }, }, ] `; @@ -71,99 +80,105 @@ exports[`generatePipelines should generate correct pipelines with only docs chan exports[`generatePipelines should generate correct pipelines with full BWC expansion 1`] = ` [ { - "steps": [ - { - "group": "bwc", - "steps": [ - { - "agents": { - "buildDirectory": "/dev/shm/bk", - "image": "family/elasticsearch-ubuntu-2004", - "machineType": "custom-32-98304", - "provider": "gcp", - }, - "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest", - "env": { - "BWC_VERSION": "7.0.0", - }, - "key": "full-bwc:7_0_0", - "label": "7.0.0 / bwc", - "timeout_in_minutes": 300, - }, - { - "agents": { - "buildDirectory": "/dev/shm/bk", - "image": "family/elasticsearch-ubuntu-2004", - "machineType": "custom-32-98304", - "provider": "gcp", - }, - "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest", - "env": { - "BWC_VERSION": "7.0.1", - }, - "key": "full-bwc:7_0_1", - "label": "7.0.1 / bwc", - "timeout_in_minutes": 300, - }, - { - "agents": { - "buildDirectory": "/dev/shm/bk", - "image": "family/elasticsearch-ubuntu-2004", - "machineType": "custom-32-98304", - "provider": "gcp", - }, - "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest", - "env": { - "BWC_VERSION": "7.1.0", + "name": "full-bwc", + "pipeline": { + "steps": [ + { + "group": "bwc", + "steps": [ + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.0#bwcTest", + "env": { + "BWC_VERSION": "7.0.0", + }, + "key": "full-bwc:7_0_0", + "label": "7.0.0 / bwc", + "timeout_in_minutes": 300, }, - "key": "full-bwc:7_1_0", - "label": "7.1.0 / bwc", - "timeout_in_minutes": 300, - }, - { - "agents": { - "buildDirectory": "/dev/shm/bk", - "image": "family/elasticsearch-ubuntu-2004", - "machineType": "custom-32-98304", - "provider": "gcp", + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.0.1#bwcTest", + "env": { + "BWC_VERSION": "7.0.1", + }, + "key": "full-bwc:7_0_1", + "label": "7.0.1 / bwc", + "timeout_in_minutes": 300, }, - "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest", - "env": { - "BWC_VERSION": "8.10.0", + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.1.0#bwcTest", + "env": { + "BWC_VERSION": "7.1.0", + }, + "key": "full-bwc:7_1_0", + "label": "7.1.0 / bwc", + "timeout_in_minutes": 300, }, - "key": "full-bwc:8_10_0", - "label": "8.10.0 / bwc", - "timeout_in_minutes": 300, - }, - { - "agents": { - "buildDirectory": "/dev/shm/bk", - "image": "family/elasticsearch-ubuntu-2004", - "machineType": "custom-32-98304", - "provider": "gcp", + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest", + "env": { + "BWC_VERSION": "8.10.0", + }, + "key": "full-bwc:8_10_0", + "label": "8.10.0 / bwc", + "timeout_in_minutes": 300, }, - "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest", - "env": { - "BWC_VERSION": "8.11.0", + { + "agents": { + "buildDirectory": "/dev/shm/bk", + "image": "family/elasticsearch-ubuntu-2004", + "machineType": "custom-32-98304", + "provider": "gcp", + }, + "command": ".ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest", + "env": { + "BWC_VERSION": "8.11.0", + }, + "key": "full-bwc:8_11_0", + "label": "8.11.0 / bwc", + "timeout_in_minutes": 300, }, - "key": "full-bwc:8_11_0", - "label": "8.11.0 / bwc", - "timeout_in_minutes": 300, - }, - ], - }, - ], + ], + }, + ], + }, }, { - "env": { - "CUSTOM_ENV_VAR": "value", - }, - "steps": [ - { - "command": "echo 'hello world'", - "label": "test-step", + "name": "using-defaults", + "pipeline": { + "env": { + "CUSTOM_ENV_VAR": "value", }, - ], + "steps": [ + { + "command": "echo 'hello world'", + "label": "test-step", + }, + ], + }, }, ] `; @@ -171,15 +186,18 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan exports[`generatePipelines should generate correct pipeline when using a trigger comment for it 1`] = ` [ { - "env": { - "CUSTOM_ENV_VAR": "value", - }, - "steps": [ - { - "command": "echo 'hello world'", - "label": "test-step", + "name": "using-defaults", + "pipeline": { + "env": { + "CUSTOM_ENV_VAR": "value", }, - ], + "steps": [ + { + "command": "echo 'hello world'", + "label": "test-step", + }, + ], + }, }, ] `; diff --git a/.buildkite/scripts/pull-request/pipeline.generate.ts b/.buildkite/scripts/pull-request/pipeline.generate.ts index 69caff990dcfe..aec50dcdaece0 100644 --- a/.buildkite/scripts/pull-request/pipeline.generate.ts +++ b/.buildkite/scripts/pull-request/pipeline.generate.ts @@ -6,13 +6,19 @@ import { generatePipelines } from "./pipeline"; const pipelines = generatePipelines(); for (const pipeline of pipelines) { - if (!process.env.CI) { - // Just for local debugging purposes + const yaml = stringify(pipeline.pipeline); + + console.log(`--- Generated pipeline: ${pipeline.name}`); + console.log(yaml); + + // Only do the pipeline upload if we're actually in CI + // This lets us run the tool locally and see the output + if (process.env.CI) { console.log(""); - console.log(stringify(pipeline)); - } else { + console.log("Uploading pipeline..."); + execSync(`buildkite-agent pipeline upload`, { - input: stringify(pipeline), + input: yaml, stdio: ["pipe", "inherit", "inherit"], }); } diff --git a/.buildkite/scripts/pull-request/pipeline.sh b/.buildkite/scripts/pull-request/pipeline.sh index 77bbc1e115430..3012308fbcace 100755 --- a/.buildkite/scripts/pull-request/pipeline.sh +++ b/.buildkite/scripts/pull-request/pipeline.sh @@ -2,5 +2,8 @@ set -euo pipefail +echo --- Installing bun npm install -g bun + +echo --- Generating pipeline bun .buildkite/scripts/pull-request/pipeline.generate.ts diff --git a/.buildkite/scripts/pull-request/pipeline.ts b/.buildkite/scripts/pull-request/pipeline.ts index a6a3df6ff2aff..600e0373d9cfc 100644 --- a/.buildkite/scripts/pull-request/pipeline.ts +++ b/.buildkite/scripts/pull-request/pipeline.ts @@ -116,6 +116,7 @@ export const generatePipelines = ( .filter((x) => x); if (!changedFiles?.length) { + console.log("Doing git fetch and getting merge-base"); const mergeBase = execSync( `git fetch origin ${process.env["GITHUB_PR_TARGET_BRANCH"]}; git merge-base origin/${process.env["GITHUB_PR_TARGET_BRANCH"]} HEAD`, { cwd: PROJECT_ROOT } @@ -123,12 +124,17 @@ export const generatePipelines = ( .toString() .trim(); + console.log(`Merge base: ${mergeBase}`); + const changedFilesOutput = execSync(`git diff --name-only ${mergeBase}`, { cwd: PROJECT_ROOT }).toString().trim(); changedFiles = changedFilesOutput .split("\n") .map((x) => x.trim()) .filter((x) => x); + + console.log("Changed files (first 50):"); + console.log(changedFiles.slice(0, 50).join("\n")); } let filters: ((pipeline: EsPipeline) => boolean)[] = [ @@ -154,9 +160,9 @@ export const generatePipelines = ( pipelines.sort((a, b) => (a.name ?? "").localeCompare(b.name ?? "")); const finalPipelines = pipelines.map((pipeline) => { - const finalPipeline = { ...pipeline }; - delete finalPipeline.config; - delete finalPipeline.name; + const finalPipeline = { name: pipeline.name, pipeline: { ...pipeline } }; + delete finalPipeline.pipeline.config; + delete finalPipeline.pipeline.name; return finalPipeline; });