add OpenLineage support to S3ToRedshiftOperator

.github/ISSUE_TEMPLATE/airflow_bug_report.yml

@@ -25,7 +25,7 @@ body:
         the latest release or main to see if the issue is fixed before reporting it.
       multiple: false
       options:
-        - "2.10.0b1"
+        - "2.10.0rc1"
         - "2.9.3"
         - "main (development)"
         - "Other Airflow 2 version (please specify below)"

.github/boring-cyborg.yml

@@ -549,12 +549,9 @@ labelPRBasedOnFilePath:
   area:API:
     - airflow/api/**/*
     - airflow/api_connexion/**/*
-    - airflow/www/api/**/*
     - clients/**/*
-    - docs/apache-airflow/deprecated-rest-api-ref.rst
     - docs/apache-airflow/stable-rest-api-ref.rst
     - tests/api_connexion/**/*
-    - tests/www/api/**/*
 
   area:dev-tools:
     - scripts/**/*

.github/workflows/build-images.yml

@@ -241,14 +241,14 @@ jobs:
       pull-request-target: "true"
       is-committer-build: ${{ needs.build-info.outputs.is-committer-build }}
       push-image: "true"
-      use-uv: ${{ needs.build-info.outputs.default-branch == 'main' && 'true' || 'false' }}
+      use-uv: "true"
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       platform: "linux/amd64"
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
-      build-provider-packages: "true"
+      build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
       docker-cache: ${{ needs.build-info.outputs.docker-cache }}

.github/workflows/check-providers.yml

@@ -28,6 +28,10 @@ on:  # yamllint disable-line rule:truthy
         description: "Tag to set for the image"
         required: true
         type: string
+      canary-run:
+        description: "Whether this is a canary run"
+        required: true
+        type: string
       default-python-version:
         description: "Which version of python should be used by default"
         required: true
@@ -209,6 +213,7 @@ jobs:
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       VERBOSE: "true"
+      CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
     if: inputs.skip-provider-tests != 'true'
     steps:
       - name: "Cleanup repo"

.github/workflows/ci-image-build.yml

@@ -60,7 +60,7 @@ on:  # yamllint disable-line rule:truthy
         default: "true"
         type: string
       debian-version:
-        description: "Base Debian distribution to use for the build (bookworm/bullseye)"
+        description: "Base Debian distribution to use for the build (bookworm)"
         type: string
         default: "bookworm"
       install-mysql-client-type:

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ on:  # yamllint disable-line rule:truthy
   push:
     branches: ['v[0-9]+-[0-9]+-test']
   pull_request:
-    branches: ['main']
+    branches: ['main', 'v[0-9]+-[0-9]+-test']
   workflow_dispatch:
 permissions:
   # All other permissions are set to none
@@ -319,6 +319,7 @@ jobs:
     with:
       runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
+      canary-run: ${{ needs.build-info.outputs.canary-run }}
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }}
@@ -541,7 +542,7 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       push-image: "true"
-      use-uv: ${{ needs.build-info.outputs.default-branch == 'main' && 'true' || 'false' }}
+      use-uv: "true"
       build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}

.github/workflows/news-fragment.yml

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: CI
+
+on:  # yamllint disable-line rule:truthy
+  pull_request:
+    types: [labeled, unlabeled, opened, reopened, synchronize]
+
+jobs:
+  check-news-fragment:
+    name: Check News Fragment
+    runs-on: ubuntu-20.04
+    if: "contains(github.event.pull_request.labels.*.name, 'airflow3.0:breaking')"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          # `towncrier check` runs `git diff --name-only origin/main...`, which
+          # needs a non-shallow clone.
+          fetch-depth: 0
+
+      - name: Check news fragment
+        run: >
+          pipx run towncrier check
+          --dir .
+          --config newsfragments/config.toml
+          --compare-with origin/${{ github.base_ref }}
+          ||
+          {
+          printf "\033[1;33mMissing significant newsfragment for PR labeled with
+          'airflow3.0:breaking'.\nCheck
+          https://github.com/apache/airflow/blob/main/contributing-docs/16_contribution_workflow.rst
+          for guidance.\033[m\n"
+          &&
+          false
+          ; }

.github/workflows/prod-image-build.yml

@@ -63,7 +63,7 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
       debian-version:
-        description: "Base Debian distribution to use for the build (bookworm/bullseye)"
+        description: "Base Debian distribution to use for the build (bookworm)"
         type: string
         default: "bookworm"
       install-mysql-client-type:

.github/workflows/prod-image-extra-checks.yml

@@ -64,27 +64,6 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
 jobs:
-  bullseye-image:
-    uses: ./.github/workflows/prod-image-build.yml
-    with:
-      runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
-      build-type: "Bullseye"
-      upload-package-artifact: "false"
-      image-tag: bullseye-${{ inputs.image-tag }}
-      debian-version: "bullseye"
-      python-versions: ${{ inputs.python-versions }}
-      default-python-version: ${{ inputs.default-python-version }}
-      platform: "linux/amd64"
-      branch: ${{ inputs.branch }}
-      # Always build images during the extra checks and never push them
-      push-image: "false"
-      use-uv: ${{ inputs.use-uv }}
-      build-provider-packages: ${{ inputs.build-provider-packages }}
-      upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
-      chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
-      constraints-branch: ${{ inputs.constraints-branch }}
-      docker-cache: ${{ inputs.docker-cache }}
-
   myssql-client-image:
     uses: ./.github/workflows/prod-image-build.yml
     with:

.github/workflows/push-image-cache.yml

@@ -41,7 +41,7 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
       debian-version:
-        description: "Base Debian distribution to use for the build (bookworm/bullseye)"
+        description: "Base Debian distribution to use for the build (bookworm)"
         type: string
         default: "bookworm"
       install-mysql-client-type:

.github/workflows/static-checks-mypy-docs.yml

@@ -243,7 +243,9 @@ jobs:
       INCLUDE_SUCCESS_OUTPUTS: "${{ inputs.include-success-outputs }}"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERBOSE: "true"
-    if: inputs.canary-run == 'true' && inputs.branch == 'main'
+      # Temporary disabled it until self-hosted ASF runners will be working again
+    if: false
+    # if: inputs.canary-run == 'true' && inputs.branch == 'main'
     steps:
       - name: "Cleanup repo"
         shell: bash

Dockerfile

@@ -124,11 +124,7 @@ function get_runtime_apt_deps() {
     echo
     echo "DEBIAN CODENAME: ${debian_version}"
     echo
-    if [[ "${debian_version}" == "bullseye" ]]; then
-        debian_version_apt_deps="libffi7 libldap-2.4-2 libssl1.1 netcat"
-    else
-        debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
-    fi
+    debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
     echo
     echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}"
     echo
@@ -177,19 +173,6 @@ function install_debian_dev_dependencies() {
     echo
     echo "DEBIAN CODENAME: ${debian_version}"
     echo
-    if [[ "${debian_version}" == "bullseye" ]]; then
-        echo
-        echo "Bullseye detected - replacing dependencies in additional dev apt deps"
-        echo
-        # Replace dependencies in additional dev apt deps to be compatible with Bullseye
-        ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//libgcc-11-dev/libgcc-10-dev}
-        ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//netcat-openbsd/netcat}
-        echo
-        echo "Replaced bullseye dev apt dependencies"
-        echo "${ADDITIONAL_DEV_APT_COMMAND}"
-        echo
-    fi
-
     # shellcheck disable=SC2086
     apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS}
 }

Dockerfile.ci

@@ -70,11 +70,7 @@ function get_runtime_apt_deps() {
     echo
     echo "DEBIAN CODENAME: ${debian_version}"
     echo
-    if [[ "${debian_version}" == "bullseye" ]]; then
-        debian_version_apt_deps="libffi7 libldap-2.4-2 libssl1.1 netcat"
-    else
-        debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
-    fi
+    debian_version_apt_deps="libffi8 libldap-2.5-0 libssl3 netcat-openbsd"
     echo
     echo "APPLIED INSTALLATION CONFIGURATION FOR DEBIAN VERSION: ${debian_version}"
     echo
@@ -123,19 +119,6 @@ function install_debian_dev_dependencies() {
     echo
     echo "DEBIAN CODENAME: ${debian_version}"
     echo
-    if [[ "${debian_version}" == "bullseye" ]]; then
-        echo
-        echo "Bullseye detected - replacing dependencies in additional dev apt deps"
-        echo
-        # Replace dependencies in additional dev apt deps to be compatible with Bullseye
-        ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//libgcc-11-dev/libgcc-10-dev}
-        ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS//netcat-openbsd/netcat}
-        echo
-        echo "Replaced bullseye dev apt dependencies"
-        echo "${ADDITIONAL_DEV_APT_COMMAND}"
-        echo
-    fi
-
     # shellcheck disable=SC2086
     apt-get install -y --no-install-recommends ${DEV_APT_DEPS} ${ADDITIONAL_DEV_APT_DEPS}
 }
@@ -997,15 +980,23 @@ function determine_airflow_to_use() {
             echo
             exit 0
         fi
+        if [[ ${CLEAN_AIRFLOW_INSTALLATION=} == "true" ]]; then
+            echo
+            echo "${COLOR_BLUE}Uninstalling all packages first${COLOR_RESET}"
+            echo
+            pip freeze | grep -ve "^-e" | grep -ve "^#" | grep -ve "^uv" | xargs pip uninstall -y --root-user-action ignore
+            # Now install rich ad click first to use the installation script
+            uv pip install rich rich-click click --python "/usr/local/bin/python" \
+                --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
+        fi
         python "${IN_CONTAINER_DIR}/install_airflow_and_providers.py"
+        echo
+        echo "${COLOR_BLUE}Reinstalling all development dependencies${COLOR_RESET}"
+        echo
+        python "${IN_CONTAINER_DIR}/install_devel_deps.py" \
+           --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
         # Some packages might leave legacy typing module which causes test issues
         pip uninstall -y typing || true
-        # Upgrade pytest and pytest extensions to latest version if they have been accidentally
-        # downgraded by constraints
-        pip install --upgrade pytest pytest aiofiles aioresponses pytest-asyncio pytest-custom-exit-code \
-           pytest-icdiff pytest-instafail pytest-mock pytest-rerunfailures pytest-timeouts \
-           pytest-xdist pytest requests_mock time-machine \
-           --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt
     fi
 
     if [[ "${USE_AIRFLOW_VERSION}" =~ ^2\.2\..*|^2\.1\..*|^2\.0\..* && "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=}" != "" ]]; then
@@ -1068,8 +1059,9 @@ function check_pydantic() {
         echo
         echo "${COLOR_YELLOW}Downgrading Pydantic to < 2${COLOR_RESET}"
         echo
+        # Pydantic 1.10.17/1.10.15 conflicts with aws-sam-translator so we need to exclude it
         # shellcheck disable=SC2086
-        ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0"
+        ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "pydantic<2.0.0,!=1.10.17,!=1.10.15"
         pip check
     else
         echo

INSTALL

@@ -256,9 +256,9 @@ Those extras are available as regular core airflow extras - they install optiona
 
 # START CORE EXTRAS HERE
 
-aiobotocore, apache-atlas, apache-webhdfs, async, cgroups, cloudpickle, deprecated-api, github-
-enterprise, google-auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, pydantic,
-rabbitmq, s3fs, saml, sentry, statsd, uv, virtualenv
+aiobotocore, apache-atlas, apache-webhdfs, async, cgroups, cloudpickle, github-enterprise, google-
+auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, pydantic, rabbitmq, s3fs, saml,
+sentry, statsd, uv, virtualenv
 
 # END CORE EXTRAS HERE
 

README.md

@@ -122,9 +122,7 @@ The work to add Windows support is tracked via [#10388](https://github.com/apach
 it is not a high priority. You should only use Linux-based distros as "Production" execution environment
 as this is the only environment that is supported. The only distro that is used in our CI tests and that
 is used in the [Community managed DockerHub image](https://hub.docker.com/p/apache/airflow) is
-`Debian Bookworm`. We also have support for legacy ``Debian Bullseye`` base image if you want to build a
-custom image but it is deprecated and option to do it will be removed in the Dockerfile that
-will accompany Airflow 2.9.3 so you are advised to switch to ``Debian Bookworm`` for your custom images.
+`Debian Bookworm`.
 
 <!-- END Requirements, please keep comment here to allow auto update of PyPI readme.md -->
 <!-- START Getting started, please keep comment here to allow auto update of PyPI readme.md -->
@@ -347,13 +345,9 @@ building and testing the OS version. Approximately 6 months before the end-of-re
 previous stable version of the OS, Airflow switches the images released to use the latest supported
 version of the OS.
 
-For example since ``Debian Buster`` end-of-life was August 2022, Airflow switched the images in `main` branch
-to use ``Debian Bullseye`` in February/March 2022. The version was used in the next MINOR release after
-the switch happened. In case of the Bullseye switch - 2.3.0 version used ``Debian Bullseye``.
-The images released  in the previous MINOR version continue to use the version that all other releases
-for the MINOR version used. Similar switch from ``Debian Bullseye`` to ``Debian Bookworm`` has been implemented
+For example switch from ``Debian Bullseye`` to ``Debian Bookworm`` has been implemented
 before 2.8.0 release in October 2023 and ``Debian Bookworm`` will be the only option supported as of
-Airflow 2.9.0.
+Airflow 2.10.0.
 
 Users will continue to be able to build their images using stable Debian releases until the end of regular
 support and building and verifying of the images happens in our CI but no unit tests were executed using

RELEASE_NOTES.rst

@@ -3674,8 +3674,6 @@ Details in the `SQLAlchemy Changelog <https://docs.sqlalchemy.org/en/14/changelo
 
 Previously, only one backend was used to authorize use of the REST API. In 2.3 this was changed to support multiple backends, separated by comma. Each will be tried in turn until a successful response is returned.
 
-This setting is also used for the deprecated experimental API, which only uses the first option even if multiple are given.
-
 ``airflow.models.base.Operator`` is removed (#21505)
 """"""""""""""""""""""""""""""""""""""""""""""""""""
 

airflow/__init__.py

@@ -17,7 +17,7 @@
 # under the License.
 from __future__ import annotations
 
-__version__ = "2.10.0.dev0"
+__version__ = "3.0.0.dev0"
 
 import os
 import sys

airflow/api/client/__init__.py

@@ -19,27 +19,19 @@
 
 from __future__ import annotations
 
-from importlib import import_module
-from typing import TYPE_CHECKING
-
 from airflow import api
-from airflow.configuration import conf
-
-if TYPE_CHECKING:
-    from airflow.api.client.api_client import Client
+from airflow.api.client.local_client import Client
 
 
 def get_current_api_client() -> Client:
     """Return current API Client based on current Airflow configuration."""
-    api_module = import_module(conf.get_mandatory_value("cli", "api_client"))
     auth_backends = api.load_auth()
     session = None
     for backend in auth_backends:
         session_factory = getattr(backend, "create_client_session", None)
         if session_factory:
             session = session_factory()
-        api_client = api_module.Client(
-            api_base_url=conf.get("cli", "endpoint_url"),
+        api_client = Client(
             auth=getattr(backend, "CLIENT_AUTH", None),
             session=session,
         )