Build Images #698
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| name: "Build Images" | |
| on: # yamllint disable-line rule:truthy | |
| schedule: | |
| - cron: '28 0 * * *' | |
| pull_request_target: | |
| push: | |
| branches: ['main', 'v[0-9]+-[0-9]+-test'] | |
| permissions: | |
| # all other permissions are set to none | |
| contents: read | |
| env: | |
| MOUNT_SELECTED_LOCAL_SOURCES: "false" | |
| FORCE_ANSWER_TO_QUESTIONS: "yes" | |
| CHECK_IMAGE_FOR_REBUILD: "true" | |
| SKIP_CHECK_REMOTE_IMAGE: "true" | |
| DB_RESET: "true" | |
| VERBOSE: "true" | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the | |
| # Airflow one is going to be used | |
| CONSTRAINTS_GITHUB_REPOSITORY: >- | |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && | |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} | |
| # This token is WRITE one - pull_request_target type of events always have the WRITE token | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" | |
| INSTALL_PROVIDERS_FROM_SOURCES: "true" | |
| AIRFLOW_LOGIN_TO_GITHUB_REGISTRY: "true" | |
| GITHUB_REGISTRY_PUSH_IMAGE_TAG: ${{ github.event.pull_request.head.sha || github.sha }} | |
| concurrency: | |
| group: build-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-info: | |
| timeout-minutes: 10 | |
| name: "Build Info" | |
| runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }} | |
| env: | |
| targetBranch: ${{ github.event.pull_request.base.ref }} | |
| outputs: | |
| runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }} | |
| pythonVersions: "${{ steps.selective-checks.python-versions }}" | |
| upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} | |
| allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} | |
| defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} | |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} | |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
| image-build: ${{ steps.dynamic-outputs.outputs.image-build }} | |
| cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }} | |
| targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }} | |
| defaultBranch: ${{ steps.selective-checks.outputs.default-branch }} | |
| targetCommitSha: "${{steps.discover-pr-merge-commit.outputs.targetCommitSha || | |
| github.event.pull_request.head.sha || | |
| github.sha | |
| }}" | |
| steps: | |
| - name: Discover PR merge commit | |
| id: discover-pr-merge-commit | |
| run: | | |
| TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)" | |
| echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> $GITHUB_ENV | |
| echo "::set-output name=targetCommitSha::${TARGET_COMMIT_SHA}" | |
| if: github.event_name == 'pull_request_target' | |
| # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get | |
| # up-to-date values | |
| - name: Get latest PR labels | |
| id: get-latest-pr-labels | |
| run: | | |
| echo -n "::set-output name=pullRequestLabels::" | |
| gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query=' | |
| query($node_id: ID!, $endCursor: String) { | |
| node(id:$node_id) { | |
| ... on PullRequest { | |
| labels(first: 100, after: $endCursor) { | |
| nodes { name } | |
| pageInfo { hasNextPage endCursor } | |
| } | |
| } | |
| } | |
| }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' | |
| if: github.event_name == 'pull_request_target' | |
| # Retrieve it to be able to determine which files has changed in the incoming commit of the PR | |
| # we checkout the target commit and it's parent to be able to compare them | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ env.TARGET_COMMIT_SHA }} | |
| persist-credentials: false | |
| fetch-depth: 2 | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v2 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: env | |
| run: printenv | |
| env: | |
| dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }} | |
| PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }} | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| - name: Selective checks | |
| id: selective-checks | |
| env: | |
| PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }} | |
| run: | | |
| if [[ ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then | |
| # Run selective checks | |
| ./scripts/ci/selective_ci_checks.sh "${TARGET_COMMIT_SHA}" | |
| else | |
| # Run all checks | |
| ./scripts/ci/selective_ci_checks.sh | |
| fi | |
| - name: Compute dynamic outputs | |
| id: dynamic-outputs | |
| run: | | |
| set -x | |
| if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then | |
| echo "::set-output name=targetBranch::${targetBranch}" | |
| else | |
| # Direct push to branch, or scheduled build | |
| echo "::set-output name=targetBranch::${GITHUB_REF#refs/heads/}" | |
| fi | |
| if [[ "${{ github.event_name }}" == 'schedule' ]]; then | |
| echo "::set-output name=cacheDirective::disabled" | |
| else | |
| echo "::set-output name=cacheDirective::pulled" | |
| fi | |
| if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then | |
| echo "::set-output name=image-build::true" | |
| else | |
| echo "::set-output name=image-build::false" | |
| fi | |
| env: | |
| SELECTIVE_CHECKS_IMAGE_BUILD: ${{ steps.selective-checks.outputs.image-build }} | |
| build-ci-images: | |
| permissions: | |
| packages: write | |
| timeout-minutes: 80 | |
| name: "Build CI image ${{matrix.python-version}}" | |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} | |
| needs: [build-info] | |
| strategy: | |
| matrix: | |
| # We need to attempt to build all possible versions here because pull_request_target | |
| # event is run for both main and v1-10-tests | |
| python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} | |
| fail-fast: true | |
| if: needs.build-info.outputs.image-build == 'true' | |
| env: | |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} | |
| BACKEND: postgres | |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} | |
| outputs: ${{toJSON(needs.build-info.outputs) }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: "Retrieve DEFAULTS from the _initialization.sh" | |
| # We cannot "source" the script here because that would be a security problem (we cannot run | |
| # any code that comes from the sources coming from the PR. Therefore we extract the | |
| # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands | |
| id: defaults | |
| run: | | |
| DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ | |
| awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') | |
| echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV | |
| DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ | |
| scripts/ci/libraries/_initialization.sh | \ | |
| awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') | |
| echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV | |
| DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \ | |
| awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') | |
| echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV | |
| - name: > | |
| Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder | |
| to use ci/scripts from there. | |
| uses: actions/checkout@v2 | |
| with: | |
| path: "main-airflow" | |
| ref: "${{ needs.build-info.outputs.targetBranch }}" | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: "Setup python" | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} | |
| - name: > | |
| Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch | |
| so that the PR does not override it | |
| # We should not override those scripts which become part of the image as they will not be | |
| # changed in the image built - we should only override those that are executed to build | |
| # the image. | |
| run: | | |
| rm -rf "scripts/ci" | |
| mv "main-airflow/scripts/ci" "scripts" | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/breeze/setup* | |
| - run: python -m pip install --editable ./dev/breeze/ | |
| - name: "Free space" | |
| run: airflow-freespace | |
| - name: "Build CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" | |
| run: ./scripts/ci/images/ci_build_ci_image_on_ci.sh | |
| - name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" | |
| run: ./scripts/ci/images/ci_push_ci_images.sh | |
| build-prod-images: | |
| permissions: | |
| packages: write | |
| timeout-minutes: 80 | |
| name: "Build PROD image ${{matrix.python-version}}" | |
| runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} | |
| needs: [build-info, build-ci-images] | |
| strategy: | |
| matrix: | |
| # We need to attempt to build all possible versions here because pull_request_target | |
| # event is run for both main and v1-10-tests | |
| python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} | |
| fail-fast: true | |
| if: needs.build-info.outputs.image-build == 'true' | |
| env: | |
| RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} | |
| BACKEND: postgres | |
| PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} | |
| UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} | |
| DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} | |
| VERSION_SUFFIX_FOR_PYPI: ".dev0" | |
| INSTALL_PROVIDERS_FROM_SOURCES: > | |
| ${{ needs.build-info.outputs.defaultBranch == 'main' && 'true' || 'false' }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ needs.build-info.outputs.targetCommitSha }} | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: "Retrieve DEFAULTS from the _initialization.sh" | |
| # We cannot "source" the script here because that would be a security problem (we cannot run | |
| # any code that comes from the sources coming from the PR. Therefore we extract the | |
| # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands | |
| id: defaults | |
| run: | | |
| DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ | |
| awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') | |
| echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV | |
| DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ | |
| scripts/ci/libraries/_initialization.sh | \ | |
| awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') | |
| echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV | |
| DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \ | |
| cut -d "=" -f 3 | sed s'/["}]//g') | |
| echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV | |
| - name: > | |
| Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder | |
| to use ci/scripts from there. | |
| uses: actions/checkout@v2 | |
| with: | |
| path: "main-airflow" | |
| ref: "${{ needs.build-info.outputs.targetBranch }}" | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: "Setup python" | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} | |
| - name: > | |
| Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch | |
| so that the PR does not override it | |
| # We should not override those scripts which become part of the image as they will not be | |
| # changed in the image built - we should only override those that are executed to build | |
| # the image. | |
| run: | | |
| rm -rf "scripts/ci" | |
| mv "main-airflow/scripts/ci" "scripts" | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/breeze/setup* | |
| - run: python -m pip install --editable ./dev/breeze/ | |
| - name: "Free space" | |
| run: airflow-freespace | |
| - name: "Pull CI image for PROD ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" | |
| run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh | |
| env: | |
| GITHUB_REGISTRY_PULL_IMAGE_TAG: ${{ github.event.pull_request.head.sha || github.sha }} | |
| - name: "Build PROD image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" | |
| run: ./scripts/ci/images/ci_build_prod_image_on_ci.sh | |
| - name: "Push PROD image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" | |
| run: ./scripts/ci/images/ci_push_production_images.sh | |
| cancel-on-ci-build: | |
| permissions: | |
| actions: write | |
| name: "Cancel `Tests` jobs on workflow failed/cancelled" | |
| runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }} | |
| if: failure() || cancelled() | |
| needs: [build-ci-images, build-prod-images] | |
| env: | |
| branch: ${{ github.event.pull_request.head.ref }} | |
| thisRun: ${{ github.run_id }} | |
| steps: | |
| - name: "Find running `Tests` jobs for ${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}" | |
| run: | | |
| if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then | |
| event_filter="event=pull_request&" | |
| else | |
| branch="${GITHUB_REF#refs/heads/}" | |
| event_filter="" | |
| fi | |
| for cancel_url in $( | |
| gh api "/repos/$GITHUB_REPOSITORY/actions/runs?${event_filter}branch=${branch}" \ | |
| jq -r ' | |
| .workflow_runs[] | | |
| select(.head_sha == $ENV.GITHUB_REGISTRY_PUSH_IMAGE_TAG and .status != "completed") | | |
| .cancel_url | |
| ' \ | |
| ); do | |
| # One of these URls will be _this_ workflow, so lets exclude that! | |
| [[ $cancel_url == */$thisRun/* ]] && continue | |
| echo "Cancelling $cancel_url" | |
| gh api -X POST --silent "$cancel_url" | |
| done |