From b5202fa4ef16c97e59a8848fc04995bdf5bda418 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 2 Jul 2022 16:43:22 +0200 Subject: [PATCH] Unified "dash-name" convention for outputs in ci workflows. There were errors with retieving constraints branch caused by using different convention for output names (sometimes dash, sometimes camelCase as suggested by most GitHub documents). The "dash-name" looks much better and is far more readable so we shoud unify all internal outputs to follow it. During that rename some old, unused outputs were removed, also it turned out that the new selective-check can replace previous "dynamic outputs" written in Bash as well. Additionally, the "defaults" are now retrieved via Python script, not bash script which will make it much more readable - both build_images and ci.yaml use it in the right place - before replacing the scripts and dev with the version coming in from PR in case of build_images.yaml. (cherry picked from commit 017507be1e1dbf39abcc94a44fab8869037893ea) --- .github/workflows/build-images.yml | 265 +++--- .github/workflows/ci.yml | 781 +++++++++--------- .../src/airflow_breeze/branch_defaults.py | 1 + .../airflow_breeze/utils/selective_checks.py | 4 + 4 files changed, 545 insertions(+), 506 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 2d895e3c092fd..df53d8986c809 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -27,7 +27,6 @@ env: ANSWER: "yes" CHECK_IMAGE_FOR_REBUILD: "true" SKIP_CHECK_REMOTE_IMAGE: "true" - DEBIAN_VERSION: "bullseye" DB_RESET: "true" VERBOSE: "true" GITHUB_REPOSITORY: ${{ github.repository }} @@ -51,26 +50,22 @@ jobs: name: "Build Info" runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }} env: - targetBranch: ${{ github.event.pull_request.base.ref }} - DEFAULT_BRANCH: ${{ steps.selective-checks.outputs.default-constraints-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ steps.selective-checks.outputs.default-constraints-branch }} - DEBIAN_VERSION: ${{ steps.selective-checks.outputs.debian-version }} + TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} outputs: - runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }} - pythonVersions: "${{ steps.selective-checks.python-versions }}" - upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} - allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} - allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} - defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} + runs-on: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }} + python-versions: "${{ steps.selective-checks.python-versions }}" + upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} + all-python-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} + default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} - image-build: ${{ steps.dynamic-outputs.outputs.image-build }} - cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }} - targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }} - defaultBranch: ${{ steps.selective-checks.outputs.default-branch }} - defaultConstraintsBranch: ${{ steps.selective-checks.outputs.default-constraints-branch }} - debianVersion: ${{ steps.selective-checks.outputs.debian-version }} - targetCommitSha: "${{steps.discover-pr-merge-commit.outputs.targetCommitSha || + image-build: ${{ steps.selective-checks.outputs.image-build }} + cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} + default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} + debian-version: ${{ steps.selective-checks.outputs.debian-version }} + target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha || github.event.pull_request.head.sha || github.sha }}" @@ -80,14 +75,14 @@ jobs: run: | TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)" echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> $GITHUB_ENV - echo "::set-output name=targetCommitSha::${TARGET_COMMIT_SHA}" + echo "::set-output name=target-commit-sha::${TARGET_COMMIT_SHA}" if: github.event_name == 'pull_request_target' # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get # up-to-date values - name: Get latest PR labels id: get-latest-pr-labels run: | - echo -n "::set-output name=pullRequestLabels::" + echo -n "::set-output name=pull-request-labels::" gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query=' query($node_id: ID!, $endCursor: String) { node(id:$node_id) { @@ -109,80 +104,69 @@ jobs: ref: ${{ env.TARGET_COMMIT_SHA }} persist-credentials: false fetch-depth: 2 - - name: "Retrieve DEFAULTS from the _initialization.sh" - # We cannot "source" the script here because that would be a security problem (we cannot run - # any code that comes from the sources coming from the PR. Therefore we extract the - # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands - id: defaults - run: | - DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV - DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ - scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV - DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \ - cut -d "=" -f 3 | sed s'/["}]//g') - echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV - - name: Checkout main branch to 'main-airflow' folder to use breeze from there. - uses: actions/checkout@v2 - with: - path: "main-airflow" - ref: "main" - persist-credentials: false - submodules: recursive - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: # We do not have output from selective checks yet, so we need to hardcode python python-version: 3.7 cache: 'pip' cache-dependency-path: ./dev/breeze/setup* + - name: "Retrieve defaults from branch_defaults.py" + # We cannot "execute" the branch_defaults.py python code here because that would be + # a security problem (we cannot run any code that comes from the sources coming from the PR. + # Therefore, we extract the branches via embedded Python code + # we need to do it before next step replaces checked-out breeze and scripts code coming from + # the PR, because the PR defaults have to be retrieved here. + id: defaults + run: | + python - <>$GITHUB_ENV + from pathlib import Path + import re + import sys + + DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() + BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' + CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' + DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$' + + branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) + constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) + debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) + + output = f""" + DEFAULT_BRANCH={branch} + DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} + DEBIAN_VERSION={debian_version} + """.strip() + + print(output) + # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log + print(output, file=sys.stderr) + EOF + - name: Checkout main branch to 'main-airflow' folder to use breeze from there. + uses: actions/checkout@v3 + with: + persist-credentials: false + submodules: recursive - run: ./scripts/ci/install_breeze.sh - name: Selective checks id: selective-checks env: - PR_LABELS: "$${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }}" + PR_LABELS: "$${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}" COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}" run: breeze selective-check - - name: Compute dynamic outputs - id: dynamic-outputs - run: | - set -x - if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then - echo "::set-output name=targetBranch::${targetBranch}" - else - # Direct push to branch, or scheduled build - echo "::set-output name=targetBranch::${GITHUB_REF#refs/heads/}" - fi - - if [[ "${{ github.event_name }}" == 'schedule' ]]; then - echo "::set-output name=cacheDirective::disabled" - else - echo "::set-output name=cacheDirective:registry" - fi - - if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then - echo "::set-output name=image-build::true" - else - echo "::set-output name=image-build::false" - fi - env: - SELECTIVE_CHECKS_IMAGE_BUILD: ${{ steps.selective-checks.outputs.image-build }} - name: env run: printenv env: - dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }} - PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }} + PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }} GITHUB_CONTEXT: ${{ toJson(github) }} build-ci-images: permissions: packages: write timeout-minutes: 80 - name: "Build CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + name: "Build CI images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}" + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info] if: | needs.build-info.outputs.image-build == 'true' && @@ -191,30 +175,32 @@ jobs: DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} BACKEND: sqlite outputs: ${{toJSON(needs.build-info.outputs) }} steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} + ref: ${{ needs.build-info.outputs.target-commit-sha }} persist-credentials: false submodules: recursive - - name: Checkout main branch to 'main-airflow' folder to use ci/scripts from there. - uses: actions/checkout@v2 + - name: "Setup python" + uses: actions/setup-python@v4 + with: + python-version: ${{ needs.build-info.outputs.default-python-version }} + - name: > + Checkout "main branch to 'main-airflow' folder + to use ci/scripts from there. + uses: actions/checkout@v3 with: path: "main-airflow" ref: "main" persist-credentials: false submodules: recursive - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - name: > - Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch + Override "scripts/ci" with the "main" branch so that the PR does not override it # We should not override those scripts which become part of the image as they will not be # changed in the image built - we should only override those that are executed to build @@ -224,23 +210,35 @@ jobs: rm -rfv "dev" mv -v "main-airflow/scripts/ci" "scripts" mv -v "main-airflow/dev" "." - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space - - name: > + - name: Cache pre-commit envs + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: > + breeze static-checks --type update-providers-dependencies --all-files + --show-diff-on-failure --color always || true + if: needs.build-info.outputs.default-branch == 'main' + - name: >- Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: breeze build-image --push-image --tag-as-latest --run-in-parallel env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: failure() || cancelled() run: breeze build-image --push-image --empty-image --run-in-parallel @@ -250,7 +248,7 @@ jobs: if: failure() || cancelled() run: > breeze find-newer-dependencies --max-age 1 - --python "${{ needs.build-info.outputs.defaultPythonVersion }}" + --python "${{ needs.build-info.outputs.default-python-version }}" - name: "Fix ownership" run: breeze fix-ownership if: always() @@ -259,8 +257,8 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "Build PROD images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + name: "Build PROD images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}" + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, build-ci-images] if: | needs.build-info.outputs.image-build == 'true' && @@ -269,31 +267,31 @@ jobs: DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} BACKEND: sqlite steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} + ref: ${{ needs.build-info.outputs.target-commit-sha }} persist-credentials: false submodules: recursive + - name: "Setup python" + uses: actions/setup-python@v4 + with: + python-version: ${{ needs.build-info.outputs.default-python-version }} - name: > - Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder + Checkout "main" branch to 'main-airflow' folder to use ci/scripts from there. - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: "main-airflow" - ref: "${{ needs.build-info.outputs.targetBranch }}" + ref: "main" persist-credentials: false submodules: recursive - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - name: > - Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch + Override "scripts/ci" with the "main" branch so that the PR does not override it # We should not override those scripts which become part of the image as they will not be # changed in the image built - we should only override those that are executed to build @@ -303,21 +301,34 @@ jobs: rm -rfv "dev" mv -v "main-airflow/scripts/ci" "scripts" mv -v "main-airflow/dev" "." - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space + - name: Cache pre-commit envs + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} + if: needs.build-info.outputs.default-branch == 'main' + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: > + breeze static-checks --type update-providers-dependencies --all-files + --show-diff-on-failure --color always || true + if: needs.build-info.outputs.default-branch == 'main' - name: > Pull CI image for PROD build: - ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: # Always use default Python version of CI image for preparing packages - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* @@ -331,9 +342,9 @@ jobs: run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files - - name: > + - name: >- Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: > breeze build-prod-image --run-in-parallel @@ -343,10 +354,10 @@ jobs: --disable-airflow-repo-cache --airflow-is-in-context env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: failure() || cancelled() run: breeze build-prod-image --cleanup-context --push-image --empty-image --run-in-parallel @@ -358,19 +369,19 @@ jobs: build-ci-images-arm: - timeout-minutes: 80 - name: "Build ARM CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + timeout-minutes: 120 + name: "Build ARM CI images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}" + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, build-prod-images] if: | needs.build-info.outputs.image-build == 'true' && - needs.build-info.outputs.upgradeToNewerDependencies != 'false' && + needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' && github.event.pull_request.head.repo.full_name != 'apache/airflow' env: DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} BACKEND: sqlite outputs: ${{toJSON(needs.build-info.outputs) }} steps: @@ -378,20 +389,20 @@ jobs: run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - uses: actions/checkout@v2 with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} + ref: ${{ needs.build-info.outputs.target-commit-sha }} persist-credentials: false submodules: recursive - name: > - Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder + Checkout "main" branch to 'main-airflow' folder to use ci/scripts from there. uses: actions/checkout@v2 with: path: "main-airflow" - ref: "${{ needs.build-info.outputs.targetBranch }}" + ref: "main" persist-credentials: false submodules: recursive - name: > - Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch + Override "scripts/ci" with the "main" branch so that the PR does not override it # We should not override those scripts which become part of the image as they will not be # changed in the image built - we should only override those that are executed to build @@ -404,7 +415,7 @@ jobs: - name: "Setup python" uses: actions/setup-python@v2 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space @@ -412,15 +423,15 @@ jobs: run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - name: > Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: > breeze build-image --run-in-parallel --parallelism 1 --builder airflow_cache --platform "linux/arm64" env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad6193f1f77fa..33cbb0b7515a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,36 +125,28 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} outputs: - defaultBranch: ${{ steps.selective-checks.outputs.default-branch }} - defaultConstraintsBranch: ${{ steps.selective-checks.outputs.default-constraints-branch }} - debianVersion: ${{ steps.selective-checks.outputs.debian-version }} - cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }} - waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }} - allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} - upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} - pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} - pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} - allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} - defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} - kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }} - kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} - defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }} - kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }} - defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }} - postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }} - defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }} - mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }} - mssqlVersions: ${{ steps.selective-checks.outputs.mssql-versions }} - defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }} - helmVersions: ${{ steps.selective-checks.outputs.helm-versions }} - defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }} - kindVersions: ${{ steps.selective-checks.outputs.kind-versions }} - defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }} - testTypes: ${{ steps.selective-checks.outputs.test-types }} - postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }} - mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }} - mssqlExclude: ${{ steps.selective-checks.outputs.mssql-exclude }} - sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} + debian-version: ${{ steps.selective-checks.outputs.debian-version }} + cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} + upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} + python-versions: ${{ steps.selective-checks.outputs.python-versions }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + all-python-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} + default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} + kubernetes-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} + postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} + default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} + mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} + mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }} + default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} + default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} + default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} + test-types: ${{ steps.selective-checks.outputs.test-types }} + postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} + mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} + mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }} + sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} @@ -166,21 +158,21 @@ jobs: needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} docs-filter: ${{ steps.selective-checks.outputs.docs-filter }} skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} - sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }} - pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }} - pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }} - runsOn: ${{ steps.set-runs-on.outputs.runsOn }} - runCoverage: ${{ steps.set-run-coverage.outputs.runCoverage }} - inWorkflowBuild: ${{ steps.set-in-workflow-build.outputs.inWorkflowBuild }} - buildJobDescription: ${{ steps.set-in-workflow-build.outputs.buildJobDescription }} - mergeRun: ${{ steps.set-merge-run.outputs.merge-run }} + source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} + pull-request-labels: ${{ steps.source-run-info.outputs.pullRequestLabels }} + runs-on: ${{ steps.set-runs-on.outputs.runs-on }} + run-coverage: ${{ steps.set-run-coverage.outputs.run-coverage }} + in-workflow-build: ${{ steps.set-in-workflow-build.outputs.in-workflow-build }} + build-job-description: ${{ steps.set-in-workflow-build.outputs.build-job-description }} + merge-run: ${{ steps.set-merge-run.outputs.merge-run }} steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive @@ -190,39 +182,54 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - name: Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.sha }} fetch-depth: 2 persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: # We do not have output from selective checks yet, so we need to hardcode python python-version: 3.7 cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - - run: ./scripts/ci/install_breeze.sh - - name: "Retrieve DEFAULTS from the _initialization.sh" - # We cannot "source" the script here because that would be a security problem (we cannot run - # any code that comes from the sources coming from the PR. Therefore, we extract the - # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands + - name: "Retrieve defaults from branch_defaults.py" id: defaults + # We could retrieve it differently here - by just importing the variables and + # printing them from python code, however we want to have the same code as used in + # the build-images.yml (there we cannot import python code coming from the PR - we need to + # treat the python code as text and extract the variables from there. run: | - DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV - DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \ - scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV - DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \ - awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g') - echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV + python - <>$GITHUB_ENV + from pathlib import Path + import re + import sys + + DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text() + BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$' + CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$' + DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$' + + branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) + constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) + debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1) + + output = f""" + DEFAULT_BRANCH={branch} + DEFAULT_CONSTRAINTS_BRANCH={constraints_branch} + DEBIAN_VERSION={debian_version} + """.strip() + + print(output) + # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log + print(output, file=sys.stderr) + EOF + - run: ./scripts/ci/install_breeze.sh - name: Selective checks id: selective-checks env: - PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ steps.source-run-info.outputs.pull-request-labels }}" COMMIT_REF: "${{ github.sha }}" run: breeze selective-check # Avoid having to specify the runs-on logic every time. We use the custom @@ -231,22 +238,22 @@ jobs: - name: Set runs-on id: set-runs-on env: - PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ steps.source-run-info.outputs.pull-request-labels }}" run: | if [[ ${PR_LABELS=} == *"use public runners"* ]]; then echo "Forcing running on Public Runners via `use public runners` label" - echo "::set-output name=runsOn::\"ubuntu-20.04\"" + echo "::set-output name=runs-on::\"ubuntu-20.04\"" elif [[ ${AIRFLOW_SELF_HOSTED_RUNNER} == "" ]]; then echo "Regular PR running with Public Runner" - echo "::set-output name=runsOn::\"ubuntu-20.04\"" + echo "::set-output name=runs-on::\"ubuntu-20.04\"" else echo "Maintainer or main run running with self-hosted runner" - echo "::set-output name=runsOn::\"self-hosted\"" + echo "::set-output name=runs-on::\"self-hosted\"" fi # Avoid having to specify the coverage logic every time. - name: Set run coverage id: set-run-coverage - run: echo "::set-output name=runCoverage::true" + run: echo "::set-output name=run-coverage::true" if: > github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && github.event_name == 'push' && @@ -262,12 +269,12 @@ jobs: if [[ ${GITHUB_EVENT_NAME} == "push" || ${GITHUB_EVENT_NAME} == "push" || \ ${{steps.source-run-info.outputs.sourceHeadRepo}} == "apache/airflow" ]]; then echo "Images will be built in current workflow" - echo "::set-output name=inWorkflowBuild::true" - echo "::set-output name=buildJobDescription::Build" + echo "::set-output name=in-workflow-build::true" + echo "::set-output name=build-job-description::Build" else echo "Images will be built in pull_request_target workflow" - echo "::set-output name=inWorkflowBuild::false" - echo "::set-output name=buildJobDescription::Skip Build (pull_request_target)" + echo "::set-output name=in-workflow-build::false" + echo "::set-output name=build-job-description::Skip Build (pull_request_target)" fi - name: Determine if this is merge run id: set-merge-run @@ -279,147 +286,163 @@ jobs: github.ref_name == 'main' || startsWith(github.ref_name, 'v2') && endsWith(github.ref_name, 'test') ) - - name: Compute dynamic outputs - id: dynamic-outputs - run: | - set -x - if [[ "${{ github.event_name }}" == 'schedule' ]]; then - echo "::set-output name=cacheDirective::disabled" - else - echo "::set-output name=cacheDirective::registry" - fi - - if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then - echo "::set-output name=image-build::true" - else - echo "::set-output name=image-build::false" - fi - env: - SELECTIVE_CHECKS_IMAGE_BUILD: ${{ steps.selective-checks.outputs.image-build }} - name: env run: printenv env: - dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }} - PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }} + PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }} GITHUB_CONTEXT: ${{ toJson(github) }} build-ci-images: permissions: packages: write timeout-minutes: 80 - name: > - ${{needs.build-info.outputs.buildJobDescription}} CI images - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + name: >- + ${{needs.build-info.outputs.build-job-description}} CI images + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info] env: DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: needs.build-info.outputs.inWorkflowBuild == 'true' - - uses: actions/checkout@v2 + if: needs.build-info.outputs.in-workflow-build == 'true' + - uses: actions/checkout@v3 with: ref: ${{ needs.build-info.outputs.targetCommitSha }} persist-credentials: false submodules: recursive - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - if: needs.build-info.outputs.inWorkflowBuild == 'true' + python-version: ${{ needs.build-info.outputs.default-python-version }} + if: needs.build-info.outputs.in-workflow-build == 'true' - run: ./scripts/ci/install_breeze.sh - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Free space" run: breeze free-space - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' + - name: Cache pre-commit envs + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: > + breeze static-checks --type update-providers-dependencies --all-files + --show-diff-on-failure --color always || true + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' - name: > Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: breeze build-image --push-image --tag-as-latest --run-in-parallel env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} - if: needs.build-info.outputs.inWorkflowBuild == 'true' + PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Candidates for pip resolver backtrack triggers" if: failure() || cancelled() run: > breeze find-newer-dependencies --max-age 1 - --python "${{ needs.build-info.outputs.defaultPythonVersion }}" + --python "${{ needs.build-info.outputs.default-python-version }}" - name: "Fix ownership" run: breeze fix-ownership - if: always() && needs.build-info.outputs.inWorkflowBuild == 'true' + if: always() && needs.build-info.outputs.in-workflow-build == 'true' build-prod-images: permissions: packages: write timeout-minutes: 80 name: > - ${{needs.build-info.outputs.buildJobDescription}} PROD images - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + ${{needs.build-info.outputs.build-job-description}} PROD images + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, build-ci-images] env: DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} BACKEND: sqlite - DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} VERSION_SUFFIX_FOR_PYPI: "dev0" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: needs.build-info.outputs.inWorkflowBuild == 'true' - - uses: actions/checkout@v2 + if: needs.build-info.outputs.in-workflow-build == 'true' + - uses: actions/checkout@v3 with: ref: ${{ needs.build-info.outputs.targetCommitSha }} persist-credentials: false submodules: recursive - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - if: needs.build-info.outputs.inWorkflowBuild == 'true' + python-version: ${{ needs.build-info.outputs.default-python-version }} + if: needs.build-info.outputs.in-workflow-build == 'true' - run: ./scripts/ci/install_breeze.sh - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Free space" run: breeze free-space - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' + - name: Cache pre-commit envs + uses: actions/cache@v3 + with: + path: ~/.cache/pre-commit + key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ +${{ hashFiles('.pre-commit-config.yaml') }}" + restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + - name: "Regenerate dependencies in case they was modified manually so that we can build an image" + run: > + breeze static-checks --type update-providers-dependencies --all-files + --show-diff-on-failure --color always || true + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' - name: > Pull CI image for PROD build: - ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" + ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}" run: breeze pull-image --tag-as-latest env: # Always use default Python version of CI image for preparing packages - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Prepare providers packages" run: > breeze prepare-provider-packages --package-list-file ./scripts/ci/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Prepare airflow package" run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: > Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: > breeze build-prod-image --tag-as-latest @@ -429,29 +452,29 @@ jobs: --disable-airflow-repo-cache --airflow-is-in-context env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} - if: needs.build-info.outputs.inWorkflowBuild == 'true' + PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Fix ownership" run: breeze fix-ownership - if: always() && needs.build-info.outputs.inWorkflowBuild == 'true' + if: always() && needs.build-info.outputs.in-workflow-build == 'true' run-new-breeze-tests: timeout-minutes: 10 name: Breeze unit tests - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info] steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: persist-credentials: false - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: python -m pip install --editable ./dev/breeze/ @@ -461,18 +484,18 @@ jobs: tests-ui: timeout-minutes: 10 name: React UI tests - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info] if: needs.build-info.outputs.run-ui-tests == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup node" - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: 14 - name: "Cache eslint" @@ -488,22 +511,22 @@ jobs: tests-www: timeout-minutes: 10 name: React WWW tests - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info] if: needs.build-info.outputs.run-www-tests == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup node" - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: 14 - name: "Cache eslint" - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: 'airflow/www/node_modules' key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} @@ -516,14 +539,14 @@ jobs: test-openapi-client-generation: timeout-minutes: 10 name: "Test OpenAPI client generation" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info] if: needs.build-info.outputs.needs-api-codegen == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 2 persist-credentials: false @@ -533,7 +556,7 @@ jobs: test-examples-of-prod-image-building: timeout-minutes: 60 name: "Test examples of production image building" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info] if: needs.build-info.outputs.image-build == 'true' steps: @@ -545,9 +568,9 @@ jobs: fetch-depth: 2 persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/requirements.txt - name: "Test examples of PROD image building" @@ -558,23 +581,23 @@ jobs: wait-for-ci-images: timeout-minutes: 120 name: "Wait for CI images" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, build-ci-images] if: needs.build-info.outputs.image-build == 'true' env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} BACKEND: sqlite steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -584,7 +607,7 @@ jobs: id: wait-for-images run: breeze pull-image --run-in-parallel --verify-image --wait-for-image --tag-as-latest env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Fix ownership" run: breeze fix-ownership @@ -593,27 +616,27 @@ jobs: static-checks: timeout-minutes: 30 name: "Static checks" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} if: needs.build-info.outputs.basic-checks-only == 'false' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - name: Cache pre-commit envs - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pre-commit key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ @@ -647,33 +670,33 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" static-checks-basic-checks-only: timeout-minutes: 30 name: "Static checks: basic checks only" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info] env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} if: needs.build-info.outputs.basic-checks-only == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - name: Cache pre-commit envs - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pre-commit key: "pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}-\ ${{ hashFiles('.pre-commit-config.yaml') }}" restore-keys: pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}} - name: Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.sha }} fetch-depth: 2 @@ -701,23 +724,23 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" docs: timeout-minutes: 45 name: "Build docs" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] if: needs.build-info.outputs.docs-build == 'true' env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: ${{needs.build-info.outputs.defaultPythonVersion}} + python-version: ${{needs.build-info.outputs.default-python-version}} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -727,7 +750,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: breeze pull-image --tag-as-latest env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - uses: actions/cache@v2 + - uses: actions/cache@v3 id: cache-doc-inventories with: path: ./docs/_inventory_cache/ @@ -758,24 +781,24 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" prepare-test-provider-packages-wheel: timeout-minutes: 40 name: "Build and test provider packages wheel" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false if: needs.build-info.outputs.default-branch == 'main' - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -799,8 +822,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze verify-provider-packages --use-airflow-version wheel --use-packages-from-dist --package-format wheel env: - SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgradeToNewerDependencies }}" - - name: "Remove airflow package and replace providers with 2.1-compliant versions" + SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" + - name: "Remove airflow package and replace providers with 2.2-compliant versions" run: | rm -vf dist/apache_airflow-*.whl \ dist/apache_airflow_providers_cncf_kubernetes*.whl \ @@ -808,12 +831,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" pip download --no-deps --dest dist \ apache-airflow-providers-cncf-kubernetes==3.0.0 \ apache-airflow-providers-celery==2.1.3 - - name: "Install and test provider packages and airflow on Airflow 2.1 files" + - name: "Install and test provider packages and airflow on Airflow 2.2 files" run: > - breeze verify-provider-packages --use-airflow-version 2.1.0 - --use-packages-from-dist --package-format wheel --airflow-constraints-reference constraints-2.1.0 + breeze verify-provider-packages --use-airflow-version 2.2.0 + --use-packages-from-dist --package-format wheel --airflow-constraints-reference constraints-2.2.0 env: - # The extras below are all extras that should be installed with Airflow 2.1.0 + # The extras below are all extras that should be installed with Airflow 2.2.0 AIRFLOW_EXTRAS: "airbyte,alibaba,amazon,apache.atlas.apache.beam,apache.cassandra,apache.drill,\ apache.druid,apache.hdfs,apache.hive,apache.kylin,apache.livy,apache.pig,apache.pinot,\ apache.spark,apache.sqoop,apache.webhdfs,arangodb,asana,async,\ @@ -831,26 +854,26 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" if: always() prepare-test-provider-packages-sdist: - timeout-minutes: 40 + timeout-minutes: 80 name: "Build and test provider packages sdist" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false if: needs.build-info.outputs.default-branch == 'main' - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -878,7 +901,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze verify-provider-packages --use-airflow-version sdist --use-packages-from-dist --package-format sdist env: - SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgradeToNewerDependencies }}" + SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" - name: "Fix ownership" run: breeze fix-ownership if: always() @@ -886,15 +909,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-helm: timeout-minutes: 80 name: "Python unit tests for helm chart" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} MOUNT_SELECTED_LOCAL_SOURCES: "true" TEST_TYPES: "Helm" BACKEND: "" DB_RESET: "false" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} + PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.default-python-version}} if: > needs.build-info.outputs.needs-helm-tests == 'true' && (github.repository == 'apache/airflow' || github.event_name != 'schedule') && @@ -903,13 +926,13 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -923,25 +946,25 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Tests: Helm" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: airflow-logs-helm path: "./files/airflow_logs*" retention-days: 7 - name: "Upload container logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: container-logs-helm path: "./files/container_logs*" retention-days: 7 - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' + uses: actions/upload-artifact@v3 + if: needs.build-info.outputs.run-coverage == 'true' with: name: coverage-helm path: "./files/coverage*.xml" @@ -954,33 +977,33 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" timeout-minutes: 130 name: > Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: - ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + ${{needs.build-info.outputs.test-types}} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] strategy: matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }} + python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} + postgres-version: ${{ fromJson(needs.build-info.outputs.postgres-versions) }} + exclude: ${{ fromJson(needs.build-info.outputs.postgres-exclude) }} fail-fast: false env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} BACKEND: postgres POSTGRES_VERSION: ${{ matrix.postgres-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" + TEST_TYPES: "${{needs.build-info.outputs.test-types}}" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -994,28 +1017,28 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/testing/run_downgrade_test.sh - name: "Test Offline SQL generation" run: ./scripts/ci/testing/run_offline_sql_test.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" + - name: "Tests: ${{needs.build-info.outputs.test-types}}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}} path: "./files/airflow_logs*" retention-days: 7 - name: "Upload container logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} path: "./files/container_logs*" retention-days: 7 - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' + uses: actions/upload-artifact@v3 + if: needs.build-info.outputs.run-coverage == 'true' with: name: coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} path: "./files/coverage*.xml" @@ -1027,33 +1050,33 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-mysql: timeout-minutes: 130 name: > - MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] strategy: matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }} + python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} + mysql-version: ${{ fromJson(needs.build-info.outputs.mysql-versions) }} + exclude: ${{ fromJson(needs.build-info.outputs.mysql-exclude) }} fail-fast: false env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} BACKEND: mysql MYSQL_VERSION: ${{ matrix.mysql-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" + TEST_TYPES: "${{needs.build-info.outputs.test-types}}" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -1067,28 +1090,28 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: ./scripts/ci/testing/run_downgrade_test.sh - name: "Test Offline SQL generation" run: ./scripts/ci/testing/run_offline_sql_test.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" + - name: "Tests: ${{needs.build-info.outputs.test-types}}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}} path: "./files/airflow_logs*" retention-days: 7 - name: "Upload container logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} path: "./files/container_logs*" retention-days: 7 - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' + uses: actions/upload-artifact@v3 + if: needs.build-info.outputs.run-coverage == 'true' with: name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} path: "./files/coverage*.xml" @@ -1100,33 +1123,33 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-mssql: timeout-minutes: 130 name: > - MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] strategy: matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - mssql-version: ${{ fromJson(needs.build-info.outputs.mssqlVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.mssqlExclude) }} + python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} + mssql-version: ${{ fromJson(needs.build-info.outputs.mssql-versions) }} + exclude: ${{ fromJson(needs.build-info.outputs.mssql-exclude) }} fail-fast: false env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} BACKEND: mssql MSSQL_VERSION: ${{ matrix.mssql-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" + TEST_TYPES: "${{needs.build-info.outputs.test-types}}" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -1138,28 +1161,28 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Test downgrade" run: ./scripts/ci/testing/run_downgrade_test.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" + - name: "Tests: ${{needs.build-info.outputs.test-types}}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: airflow-logs-${{matrix.python-version}}-${{matrix.mssql-version}} path: "./files/airflow_logs*" retention-days: 7 - name: "Upload container logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: container-logs-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} path: "./files/container_logs*" retention-days: 7 - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' + uses: actions/upload-artifact@v3 + if: needs.build-info.outputs.run-coverage == 'true' with: name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} path: "./files/coverage*.xml" @@ -1171,31 +1194,31 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-sqlite: timeout-minutes: 130 name: > - Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images] strategy: matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }} + python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} + exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }} fail-fast: false env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} BACKEND: sqlite - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" + TEST_TYPES: "${{needs.build-info.outputs.test-types}}" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -1207,28 +1230,28 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Test downgrade" run: ./scripts/ci/testing/run_downgrade_test.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" + - name: "Tests: ${{needs.build-info.outputs.test-types}}" run: ./scripts/ci/testing/ci_run_airflow_testing.sh env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: airflow-logs-${{matrix.python-version}} path: './files/airflow_logs*' retention-days: 7 - name: "Upload container logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: container-logs-sqlite-${{matrix.python-version}} path: "./files/container_logs*" retention-days: 7 - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' + uses: actions/upload-artifact@v3 + if: needs.build-info.outputs.run-coverage == 'true' with: name: coverage-sqlite-${{matrix.python-version}} path: ./files/coverage*.xml @@ -1240,27 +1263,27 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-quarantined: timeout-minutes: 60 name: "Quarantined tests" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} continue-on-error: true needs: [build-info, wait-for-ci-images] env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}} - POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} + MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}} + POSTGRES_VERSION: ${{needs.build-info.outputs.default-postgres-version}} TEST_TYPES: "Quarantined" - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} if: needs.build-info.outputs.run-tests == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - name: "Set issue id for main" @@ -1285,31 +1308,31 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: "Tests: Quarantined" run: ./scripts/ci/testing/ci_run_quarantined_tests.sh env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" - name: "Upload Quarantine test results" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: always() with: name: quarantined-tests path: "files/test_result-*.xml" retention-days: 7 - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: airflow-logs-quarantined-${{ matrix.backend }} path: "./files/airflow_logs*" retention-days: 7 - name: "Upload container logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() with: name: container-logs-quarantined-${{ matrix.backend }} path: "./files/container_logs*" retention-days: 7 - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' + uses: actions/upload-artifact@v3 + if: needs.build-info.outputs.run-coverage == 'true' with: name: coverage-quarantined-${{ matrix.backend }} path: "./files/coverage*.xml" @@ -1321,7 +1344,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" upload-coverage: timeout-minutes: 15 name: "Upload coverage" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} continue-on-error: true needs: - build-info @@ -1331,19 +1354,19 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - tests-mssql - tests-quarantined env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} # Only upload coverage on merges to main - if: needs.build-info.outputs.runCoverage == 'true' + if: needs.build-info.outputs.run-coverage == 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: "Download all artifacts from the current build" - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 with: path: ./coverage-files - name: "Removes unnecessary artifacts" @@ -1356,31 +1379,31 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" wait-for-prod-images: timeout-minutes: 120 name: "Wait for PROD images" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-ci-images, build-prod-images] if: needs.build-info.outputs.image-build == 'true' env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} BACKEND: sqlite - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space - name: "Cache virtualenv environment" - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: '.build/.docker_venv' key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} @@ -1391,7 +1414,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" # For the images to be available and test them. run: breeze pull-prod-image --verify-image --wait-for-image --run-in-parallel env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Fix ownership" run: breeze fix-ownership @@ -1400,23 +1423,23 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" test-docker-compose-quick-start: timeout-minutes: 60 name: "Test docker-compose quick start" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-prod-images] if: needs.build-info.outputs.image-build == 'true' env: - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 2 persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -1435,25 +1458,25 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-kubernetes: timeout-minutes: 240 name: Helm Chart; ${{matrix.executor}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-prod-images] strategy: matrix: executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] fail-fast: false env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} BACKEND: postgres RUN_TESTS: "true" RUNTIME: "kubernetes" KUBERNETES_MODE: "image" EXECUTOR: ${{matrix.executor}} - KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" - HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" + KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}" + HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}" CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} + ${{needs.build-info.outputs.python-versions-list-as-string}} CURRENT_KUBERNETES_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.kubernetesVersionsListAsString}} + ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} if: > ( needs.build-info.outputs.run-kubernetes-tests == 'true' || needs.build-info.outputs.needs-helm-tests == 'true' ) && @@ -1462,13 +1485,13 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -1477,22 +1500,22 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-prod-image --run-in-parallel --tag-as-latest env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Cache bin folder with tools for kubernetes testing" - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ".build/kubernetes-bin" key: "kubernetes-binaries --${{ needs.build-info.outputs.defaultKindVersion }}\ --${{ needs.build-info.outputs.defaultHelmVersion }}" +-${{ needs.build-info.outputs.default-kind-version }}\ +-${{ needs.build-info.outputs.default-helm-version }}" restore-keys: "kubernetes-binaries" - name: "Kubernetes Tests" run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" - name: "Upload KinD logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() || cancelled() with: name: kind-logs-${{matrix.executor}} @@ -1505,21 +1528,21 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-helm-executor-upgrade: timeout-minutes: 150 name: Helm Chart Executor Upgrade - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: [build-info, wait-for-prod-images] env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} BACKEND: postgres RUN_TESTS: "true" RUNTIME: "kubernetes" KUBERNETES_MODE: "image" EXECUTOR: "KubernetesExecutor" - KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" - HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" + KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}" + HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}" CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} + ${{needs.build-info.outputs.python-versions-list-as-string}} CURRENT_KUBERNETES_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.kubernetesVersionsListAsString}} + ${{needs.build-info.outputs.kubernetes-versions-list-as-string}} if: > needs.build-info.outputs.run-kubernetes-tests == 'true' && needs.build-info.outputs.default-branch == 'main' @@ -1527,13 +1550,13 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -1542,33 +1565,33 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-prod-image --run-in-parallel --tag-as-latest env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Cache virtualenv for kubernetes testing" - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ".build/.kubernetes_venv" - key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\ - -${{needs.build-info.outputs.kubernetesVersionsListAsString}} - -${{needs.build-info.outputs.pythonVersionsListAsString}} + key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\ + -${{needs.build-info.outputs.kubernetes-versions-list-as-string}} + -${{needs.build-info.outputs.python-versions-list-as-string}} -${{ hashFiles('setup.py','setup.cfg') }}" - restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\ - -${{needs.build-info.outputs.kubernetesVersionsListAsString}} - -${{needs.build-info.outputs.pythonVersionsListAsString}}" + restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\ + -${{needs.build-info.outputs.kubernetes-versions-list-as-string}} + -${{needs.build-info.outputs.python-versions-list-as-string}}" - name: "Cache bin folder with tools for kubernetes testing" - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ".build/kubernetes-bin" key: "kubernetes-binaries - -${{ needs.build-info.outputs.defaultKindVersion }}\ - -${{ needs.build-info.outputs.defaultHelmVersion }}" + -${{ needs.build-info.outputs.default-kind-version }}\ + -${{ needs.build-info.outputs.default-helm-version }}" restore-keys: "kubernetes-binaries" - name: "Kubernetes Helm Chart Executor Upgrade Tests" run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" + PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}" - name: "Upload KinD logs" - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: failure() || cancelled() with: name: kind-logs-KubernetesExecutor @@ -1583,7 +1606,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" contents: write timeout-minutes: 40 name: "Constraints" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: - build-info - wait-for-ci-images @@ -1594,20 +1617,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - tests-mssql - tests-postgres env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - if: needs.build-info.outputs.upgradeToNewerDependencies != 'false' + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} + if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false submodules: recursive - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -1616,7 +1639,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --run-in-parallel --tag-as-latest env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Generate constraints" run: | @@ -1625,24 +1648,24 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints-no-providers breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints env: - PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} - name: "Set constraints branch name" id: constraints-branch run: ./scripts/ci/constraints/ci_branch_constraints.sh - if: needs.build-info.outputs.mergeRun == 'true' + if: needs.build-info.outputs.merge-run == 'true' - name: Checkout ${{ steps.constraints-branch.outputs.branch }} - uses: actions/checkout@v2 - if: needs.build-info.outputs.mergeRun == 'true' + uses: actions/checkout@v3 + if: needs.build-info.outputs.merge-run == 'true' with: path: "repo" ref: ${{ steps.constraints-branch.outputs.branch }} persist-credentials: false - - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}" + - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}" run: ./scripts/ci/constraints/ci_commit_constraints.sh - if: needs.build-info.outputs.mergeRun == 'true' + if: needs.build-info.outputs.merge-run == 'true' - name: "Push changes" uses: ./.github/actions/github-push-action - if: needs.build-info.outputs.mergeRun == 'true' + if: needs.build-info.outputs.merge-run == 'true' with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: ${{ steps.constraints-branch.outputs.branch }} @@ -1660,31 +1683,31 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" packages: write timeout-minutes: 120 name: "Push Image Cache" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: - build-info - constraints - docs - if: needs.build-info.outputs.mergeRun == 'true' + if: needs.build-info.outputs.merge-run == 'true' strategy: fail-fast: false matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} + python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} platform: ["linux/amd64", "linux/arm64"] env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }} PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: persist-credentials: false - name: "Setup python" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + python-version: ${{ needs.build-info.outputs.default-python-version }} cache: 'pip' cache-dependency-path: ./dev/breeze/setup* - run: ./scripts/ci/install_breeze.sh @@ -1696,7 +1719,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" run: breeze pull-image --tag-as-latest env: # Always use default Python version of CI image for preparing packages - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* @@ -1744,11 +1767,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" if: always() build-ci-arm-images: - timeout-minutes: 80 + timeout-minutes: 120 name: > - ${{needs.build-info.outputs.buildJobDescription}} CI ARM images - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} + ${{needs.build-info.outputs.build-job-description}} CI ARM images + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} + runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }} needs: - build-info - wait-for-ci-images @@ -1762,46 +1785,46 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }} - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} - if: needs.build-info.outputs.upgradeToNewerDependencies != 'false' + RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }} + if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - uses: actions/checkout@v2 with: ref: ${{ needs.build-info.outputs.targetCommitSha }} persist-credentials: false submodules: recursive - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Setup python" uses: actions/setup-python@v2 with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - if: needs.build-info.outputs.inWorkflowBuild == 'true' + python-version: ${{ needs.build-info.outputs.default-python-version }} + if: needs.build-info.outputs.in-workflow-build == 'true' - run: ./scripts/ci/install_breeze.sh - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Free space" run: breeze free-space - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - if: needs.build-info.outputs.inWorkflowBuild == 'true' + if: needs.build-info.outputs.in-workflow-build == 'true' - name: > Build CI ARM images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + ${{ needs.build-info.outputs.all-python-versions-list-as-string }} run: > breeze build-image --run-in-parallel --parallelism 1 --builder airflow_cache --platform "linux/arm64" env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} - if: needs.build-info.outputs.inWorkflowBuild == 'true' + PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }} + if: needs.build-info.outputs.in-workflow-build == 'true' - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh - if: always() && needs.build-info.outputs.inWorkflowBuild == 'true' + if: always() && needs.build-info.outputs.in-workflow-build == 'true' - name: "Fix ownership" run: breeze fix-ownership - if: always() && needs.build-info.outputs.inWorkflowBuild == 'true' + if: always() && needs.build-info.outputs.in-workflow-build == 'true' diff --git a/dev/breeze/src/airflow_breeze/branch_defaults.py b/dev/breeze/src/airflow_breeze/branch_defaults.py index ea837607501dc..f02f4812cf178 100644 --- a/dev/breeze/src/airflow_breeze/branch_defaults.py +++ b/dev/breeze/src/airflow_breeze/branch_defaults.py @@ -38,3 +38,4 @@ AIRFLOW_BRANCH = "v2-3-test" DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-2-3" +DEBIAN_VERSION = "bullseye" diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 57da548421b3f..f47e30ec85095 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -513,3 +513,7 @@ def docs_filter(self) -> str: @cached_property def skip_pre_commits(self) -> str: return "identity" if self._default_branch == "main" else "identity,check-airflow-2-2-compatibility" + + @cached_property + def cache_directive(self) -> str: + return "disabled" if self._github_event == GithubEvents.SCHEDULE else "registry"