Sync v2-7-stable with v2-7-test to release 2.7.1

.github/actions/build-ci-images/action.yml

@@ -34,12 +34,18 @@ runs:
     - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}"
       shell: bash
       run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure
-    - name: "Show dependencies to be upgraded"
+    - name: "Generate source constraints"
       shell: bash
       run: >
         breeze release-management generate-constraints --run-in-parallel
         --airflow-constraints-mode constraints-source-providers
       if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
+    - name: "Generate PyPI constraints"
+      shell: bash
+      run: >
+        breeze release-management generate-constraints --run-in-parallel
+        --airflow-constraints-mode constraints
+      if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' and ${{ inputs.build-provider-packages != 'true' }}
     - name: "Print dependency upgrade summary"
       shell: bash
       run: |
@@ -48,6 +54,12 @@ runs:
           cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true
         done
       if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
+    - name: "Upload constraint artifacts"
+      uses: actions/upload-artifact@v3
+      with:
+        name: constraints
+        path: ./files/constraints-*/constraints-*.txt
+        retention-days: 7
     - name: "Fix ownership"
       shell: bash
       run: breeze ci fix-ownership

.github/actions/build-prod-images/action.yml

@@ -56,13 +56,30 @@ runs:
     - name: "Move dist packages to docker-context files"
       shell: bash
       run: mv -v ./dist/*.whl ./docker-context-files
-    - name: "Build & Push PROD images ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
+    - name: "Download constraints from the CI build"
+      uses: actions/download-artifact@v3
+      with:
+        name: constraints
+        path: ./docker-context-files
+      if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
+    - name: "Build & Push PROD images with source providers ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
+      shell: bash
+      run: >
+        breeze prod-image build --tag-as-latest --run-in-parallel --push
+        --install-packages-from-context --airflow-constraints-mode constraints-source-providers
+        --use-constraints-for-context-packages
+      env:
+        COMMIT_SHA: ${{ github.sha }}
+      if: ${{ inputs.build-provider-packages == 'true' }}
+    - name: "Build & Push PROD images with PyPi providers ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
       shell: bash
       run: >
         breeze prod-image build --tag-as-latest --run-in-parallel --push
-        --install-packages-from-context --upgrade-on-failure
+        --install-packages-from-context --airflow-constraints-mode constraints
+        --use-constraints-for-context-packages
       env:
         COMMIT_SHA: ${{ github.sha }}
+      if: ${{ inputs.build-provider-packages != 'true' }}
     - name: "Fix ownership"
       shell: bash
       run: breeze ci fix-ownership

.github/workflows/build-images.yml

@@ -99,7 +99,7 @@ jobs:
             }' --jq '.data.node.labels.nodes[]' | jq --slurp -c '[.[].name]' >> ${GITHUB_OUTPUT}
         if: github.event_name == 'pull_request_target'
       # Retrieve it to be able to determine which files has changed in the incoming commit of the PR
-      # we checkout the target commit and it's parent to be able to compare them
+      # we checkout the target commit and its parent to be able to compare them
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
       - uses: actions/checkout@v3

.pre-commit-config.yaml

@@ -52,7 +52,7 @@ repos:
           - --comment-style
           - "/*||*/"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all RST files
@@ -61,7 +61,7 @@ repos:
           - --comment-style
           - "||"
           - --license-filepath
-          - license-templates/LICENSE.rst
+          - scripts/ci/license-templates/LICENSE.rst
           - --fuzzy-match-generates-todo
         files: \.rst$
       - id: insert-license
@@ -72,7 +72,7 @@ repos:
           - --comment-style
           - "/*!| *| */"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all JINJA template files
@@ -82,7 +82,7 @@ repos:
           - --comment-style
           - "{#||#}"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all Shell files
@@ -92,7 +92,7 @@ repos:
           - --comment-style
           - "|#|"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all Python files
@@ -102,7 +102,7 @@ repos:
           - --comment-style
           - "|#|"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all XML files
@@ -112,7 +112,7 @@ repos:
           - --comment-style
           - "<!--||-->"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all Helm template files
@@ -121,7 +121,7 @@ repos:
           - --comment-style
           - "{{/*||*/}}"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all YAML files except Helm templates
@@ -132,7 +132,7 @@ repos:
           - --comment-style
           - "|#|"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all Markdown files
@@ -142,7 +142,7 @@ repos:
           - --comment-style
           - "<!--|| -->"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
       - id: insert-license
         name: Add license for all other files
@@ -151,7 +151,7 @@ repos:
           - --comment-style
           - "|#|"
           - --license-filepath
-          - license-templates/LICENSE.txt
+          - scripts/ci/license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
         files: >
           \.cfg$|\.conf$|\.ini$|\.ldif$|\.properties$|\.readthedocs$|\.service$|\.tf$|Dockerfile.*$
@@ -404,7 +404,8 @@ repos:
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py
         pass_filenames: false
-        additional_dependencies: ['packaging']
+        files: ^airflow/migrations/versions|^airflow/__init__.py$
+        additional_dependencies: ['packaging','google-re2']
       - id: update-version
         name: Update version to the latest version in the documentation
         entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py
@@ -747,6 +748,12 @@ repos:
         exclude: ^tests/system/providers/google/cloud/bigquery/example_bigquery_queries\.py$
         pass_filenames: true
         additional_dependencies: ['rich>=12.4.4']
+      - id: generate-pypi-readme
+        name: Generate PyPI README
+        entry: ./scripts/ci/pre_commit/pre_commit_generate_pypi_readme.py
+        language: python
+        files: ^README\.md$
+        pass_filenames: false
       - id: lint-markdown
         name: Run markdownlint
         description: Checks the style of Markdown files.

CONTRIBUTING.rst

@@ -677,9 +677,9 @@ druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterp
 google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes,
 ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql,
 mysql, neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill,
-password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
-segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd,
-tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk
+password, pinot, plexus, postgres, presto, rabbitmq, redis, s3, salesforce, samba, segment,
+sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau,
+tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk
   .. END EXTRAS HERE
 
 Provider packages

CONTRIBUTORS_QUICK_START.rst

@@ -104,7 +104,7 @@ Colima
 ------
 If you use Colima as your container runtimes engine, please follow the next steps:
 
-1. `Install buildx manually <https://github.com/docker/buildx#manual-download>`_ and follow it's instructions
+1. `Install buildx manually <https://github.com/docker/buildx#manual-download>`_ and follow its instructions
 
 2. Link the Colima socket to the default socket path. Note that this may break other Docker servers.
 
@@ -252,7 +252,7 @@ Typical development tasks
 #########################
 
 For many of the development tasks you will need ``Breeze`` to be configured. ``Breeze`` is a development
-environment which uses docker and docker-compose and it's main purpose is to provide a consistent
+environment which uses docker and docker-compose and its main purpose is to provide a consistent
 and repeatable environment for all the contributors and CI. When using ``Breeze`` you avoid the "works for me"
 syndrome - because not only others can reproduce easily what you do, but also the CI of Airflow uses
 the same environment to run all tests - so you should be able to easily reproduce the same failures you

Dockerfile

@@ -35,7 +35,7 @@
 #                        much smaller.
 #
 # Use the same builder frontend version for everyone
-ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv"
+ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,openlineage,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv"
 ARG ADDITIONAL_AIRFLOW_EXTRAS=""
 ARG ADDITIONAL_PYTHON_DEPS=""
 
@@ -586,17 +586,42 @@ function install_airflow_and_providers_from_docker_context_files(){
         return
     fi
 
-    echo
-    echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}"
-    echo
-    # force reinstall all airflow + provider package local files with eager upgrade
-    set -x
-    pip install "${pip_flags[@]}" --root-user-action ignore --upgrade --upgrade-strategy eager \
-        ${ADDITIONAL_PIP_INSTALL_FLAGS} \
-        ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \
-        ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=}
-    set +x
+    if [[ ${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=} == "true" ]]; then
+        local python_version
+        python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+        local local_constraints_file=/docker-context-files/constraints-"${python_version}"/${AIRFLOW_CONSTRAINTS_MODE}-"${python_version}".txt
 
+        if [[ -f "${local_constraints_file}" ]]; then
+            echo
+            echo "${COLOR_BLUE}Installing docker-context-files packages with constraints found in ${local_constraints_file}${COLOR_RESET}"
+            echo
+            # force reinstall all airflow + provider packages with constraints found in
+            set -x
+            pip install "${pip_flags[@]}" --root-user-action ignore --upgrade \
+                ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \
+                ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages}
+            set +x
+        else
+            echo
+            echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}"
+            echo
+            set -x
+            pip install "${pip_flags[@]}" --root-user-action ignore \
+                ${ADDITIONAL_PIP_INSTALL_FLAGS} \
+                --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \
+                ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages}
+            set +x
+        fi
+    else
+        echo
+        echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}"
+        echo
+        set -x
+        pip install "${pip_flags[@]}" --root-user-action ignore \
+            ${ADDITIONAL_PIP_INSTALL_FLAGS} \
+            ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages}
+        set +x
+    fi
     common::install_pip_version
     pip check
 }
@@ -1269,6 +1294,12 @@ COPY --from=scripts common.sh install_pip_version.sh \
 # is installed from docker-context files rather than from PyPI)
 ARG INSTALL_PACKAGES_FROM_CONTEXT="false"
 
+# Normally constraints are not used when context packages are build - because we might have packages
+# that are conflicting with Airflow constraints, however there are cases when we want to use constraints
+# for example in CI builds when we already have source-package constraints - either from github branch or
+# from eager-upgraded constraints by the CI builds
+ARG USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES="false"
+
 # In case of Production build image segment we want to pre-install main version of airflow
 # dependencies from GitHub so that we do not have to always reinstall it from the scratch.
 # The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false")
@@ -1293,6 +1324,7 @@ ARG VERSION_SUFFIX_FOR_PYPI=""
 
 ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \
     INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \
+    USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES} \
     VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI}
 
 WORKDIR ${AIRFLOW_HOME}
@@ -1325,7 +1357,7 @@ RUN if [[ -f /docker-context-files/requirements.txt ]]; then \
 
 ##############################################################################################
 # This is the actual Airflow image - much smaller than the build one. We copy
-# installed Airflow and all it's dependencies from the build image to make it smaller.
+# installed Airflow and all its dependencies from the build image to make it smaller.
 ##############################################################################################
 FROM ${PYTHON_BASE_IMAGE} as main
 

Dockerfile.ci

@@ -1233,7 +1233,7 @@ ARG PYTHON_BASE_IMAGE
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
 
 # By increasing this number we can do force build of all dependencies
-ARG DEPENDENCIES_EPOCH_NUMBER="8"
+ARG DEPENDENCIES_EPOCH_NUMBER="9"
 
 # Make sure noninteractive debian install is used and language variables set
 ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
@@ -1317,7 +1317,7 @@ ARG AIRFLOW_CONSTRAINTS_LOCATION=""
 ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
 # By changing the epoch we can force reinstalling Airflow and pip all dependencies
 # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable.
-ARG AIRFLOW_CI_BUILD_EPOCH="4"0
+ARG AIRFLOW_CI_BUILD_EPOCH="5"
 ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
 # By default in the image, we are installing all providers when installing from sources
 ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
@@ -1377,7 +1377,7 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}"
 # force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better
 # dependency resolution and we do not need to limit the versions of the dependencies
 # aiobotocore is limited temporarily until it stops backtracking pip
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="aiobotocore<2.6.0"
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=""
 ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
 ARG VERSION_SUFFIX_FOR_PYPI=""
 

INSTALL

@@ -104,9 +104,9 @@ druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterp
 google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes,
 ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql,
 mysql, neo4j, odbc, openfaas, openlineage, opsgenie, oracle, otel, pagerduty, pandas, papermill,
-password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
-segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd,
-tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk
+password, pinot, plexus, postgres, presto, rabbitmq, redis, s3, salesforce, samba, segment,
+sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau,
+tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, zendesk
 # END EXTRAS HERE
 
 # For installing Airflow in development environments - see CONTRIBUTING.rst