From 66723b968dddd3a18d2d43111c6b3691af7b5591 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Tue, 7 Sep 2021 13:28:07 +0100 Subject: [PATCH] Build CI images for the merge result of a PR, not the tip of the PR The change to use pull_request_target had the unintended side-effect of meaning that the images were built using the tip of the PR, instead of the merge of main and the PR branch. This restores that behaviour (and should mean that fewer PRs will need rebasing to fix their builds when master gets broken) --- .github/workflows/build-images.yml | 43 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 1c0e1e24bb3dc..4d9a1177cee7c 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -46,8 +46,8 @@ env: GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false" INSTALL_PROVIDERS_FROM_SOURCES: "true" - TARGET_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} AIRFLOW_LOGIN_TO_GITHUB_REGISTRY: "true" + GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.pull_request.head.sha || github.sha }} concurrency: group: build-${{ github.event.pull_request.number || github.ref }} @@ -60,7 +60,6 @@ jobs: name: "Build Info" runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }} env: - GITHUB_CONTEXT: ${{ toJson(github) }} targetBranch: ${{ github.event.pull_request.base.ref }} pullRequestLabels: "${{ toJSON(github.event.pull_request.labels.*.name) }}" outputs: @@ -75,7 +74,18 @@ jobs: cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }} targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }} defaultBranch: ${{ steps.selective-checks.outputs.default-branch }} + targetCommitSha: "${{steps.discover-pr-merge-commit.outputs.targetCommitSha || + github.event.pull_request.head.sha || + github.sha + }}" steps: + - name: Discover PR merge commit + id: discover-pr-merge-commit + run: | + TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)" + echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> $GITHUB_ENV + echo "::set-output name=targetCommitSha::${TARGET_COMMIT_SHA}" + if: github.event_name == 'pull_request_target' # Retrieve it to be able to determine which files has changed in the incoming commit of the PR # we checkout the target commit and it's parent to be able to compare them - uses: actions/checkout@v2 @@ -92,6 +102,7 @@ jobs: run: printenv env: dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }} + GITHUB_CONTEXT: ${{ toJson(github) }} - name: Selective checks id: selective-checks env: @@ -154,13 +165,9 @@ jobs: ${{ github.event_name == 'pull_request_target' && 'false' || 'true' }} outputs: ${{toJSON(needs.build-info.outputs) }} steps: - - name: Set envs - # Can't refer to "global" env to set a var in a job's env context - run: | - echo "GITHUB_REGISTRY_PUSH_IMAGE_TAG=${TARGET_COMMIT_SHA}" >> "$GITHUB_ENV" - uses: actions/checkout@v2 with: - ref: ${{ env.TARGET_COMMIT_SHA }} + ref: ${{ needs.build-info.outputs.targetCommitSha }} persist-credentials: false submodules: recursive - name: "Retrieve DEFAULTS from the _initialization.sh" @@ -200,9 +207,9 @@ jobs: mv "main-airflow/scripts/ci" "scripts" - name: "Free space" run: ./scripts/ci/tools/free_space.sh - - name: "Build CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + - name: "Build CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh - - name: "Push CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + - name: "Push CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" run: ./scripts/ci/images/ci_push_ci_images.sh build-prod-images: @@ -231,15 +238,9 @@ jobs: INSTALL_PROVIDERS_FROM_SOURCES: > ${{ needs.build-info.outputs.defaultBranch == 'main' && 'true' || 'false' }} steps: - - name: Set envs - # Set pull image tag for CI image build, in order to pull the image pushed - # Just a moment ago by build-ci-images job - run: | - echo "GITHUB_REGISTRY_PUSH_IMAGE_TAG=${TARGET_COMMIT_SHA}" >> "$GITHUB_ENV" - echo "GITHUB_REGISTRY_PULL_IMAGE_TAG=${TARGET_COMMIT_SHA}" >> "$GITHUB_ENV" - uses: actions/checkout@v2 with: - ref: ${{ env.TARGET_COMMIT_SHA }} + ref: ${{ needs.build-info.outputs.targetCommitSha }} persist-credentials: false submodules: recursive - name: "Retrieve DEFAULTS from the _initialization.sh" @@ -279,18 +280,18 @@ jobs: mv "main-airflow/scripts/ci" "scripts" - name: "Free space" run: ./scripts/ci/tools/free_space.sh - - name: "Build CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + - name: "Build CI images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh # Pull images built in the previous step env: GITHUB_REGISTRY_WAIT_FOR_IMAGE: "true" # Here we are using PULL_IMAGE_TAG set in the environment variables above - - name: "Build PROD images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + - name: "Build PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh env: # GITHUB_REGISTRY_PULL_IMAGE_TAG is overriden to latest in order to build PROD image using "latest" GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" - - name: "Push PROD images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}" + - name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" run: ./scripts/ci/images/ci_push_production_images.sh env: # GITHUB_REGISTRY_PULL_IMAGE_TAG is overriden to latest in order to build PROD image using "latest" @@ -307,7 +308,7 @@ jobs: branch: ${{ github.event.pull_request.head.ref }} thisRun: ${{ github.run_id }} steps: - - name: "Find running `Tests` jobs for ${{ env.TARGET_COMMIT_SHA }}" + - name: "Find running `Tests` jobs for ${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" run: | if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then event_filter="event=pull_request&" @@ -320,7 +321,7 @@ jobs: gh api "/repos/$GITHUB_REPOSITORY/actions/runs?${event_filter}branch=${branch}" \ jq -r ' .workflow_runs[] | - select(.head_sha == $ENV.TARGET_COMMIT_SHA and .status != "completed") | + select(.head_sha == $ENV.GITHUB_REGISTRY_PUSH_IMAGE_TAG and .status != "completed") | .cancel_url ' \ ); do