Implement multiple API auth backends

UPDATING.md

@@ -134,6 +134,13 @@ A new `log_template` table is introduced to solve this problem. This table is sy
 
 Previously, there was an empty class `airflow.models.base.Operator` for “type hinting”. This class was never really useful for anything (everything it did could be done better with `airflow.models.baseoperator.BaseOperator`), and has been removed. If you are relying on the class’s existence, use `BaseOperator` (for concrete operators), `airflow.models.abstractoperator.AbstractOperator` (the base class of both `BaseOperator` and the AIP-42 `MappedOperator`), or `airflow.models.operator.Operator` (a union type `BaseOperator | MappedOperator` for type annotation).
 
+### `auth_backends` replaces `auth_backend` configuration setting
+
+Previously, only one backend was used to authorize use of the REST API. In 2.3 this was changed to support multiple backends, separated by whitespace. Each will be tried in turn until a successful response is returned.
+
+This setting is also used for the deprecated experimental API, which only uses
+the first option even if multiple are given.
+
 ## Airflow 2.2.3
 
 No breaking changes.

airflow/api/__init__.py

@@ -26,17 +26,20 @@
 
 
 def load_auth():
-    """Loads authentication backend"""
-    auth_backend = 'airflow.api.auth.backend.default'
+    """Loads authentication backends"""
+    auth_backends = 'airflow.api.auth.backend.default'
     try:
-        auth_backend = conf.get("api", "auth_backend")
+        auth_backends = conf.get("api", "auth_backends")
     except AirflowConfigException:
         pass
 
-    try:
-        auth_backend = import_module(auth_backend)
-        log.info("Loaded API auth backend: %s", auth_backend)
-        return auth_backend
-    except ImportError as err:
-        log.critical("Cannot import %s for API authentication due to: %s", auth_backend, err)
-        raise AirflowException(err)
+    backends = []
+    for backend in auth_backends.split():
+        try:
+            auth = import_module(backend)
+            log.info("Loaded API auth backend: %s", backend)
+            backends.append(auth)
+        except ImportError as err:
+            log.critical("Cannot import %s for API authentication due to: %s", backend, err)
+            raise AirflowException(err)
+    return backends

airflow/api/auth/backend/session.py

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Session authentication backend"""
+from functools import wraps
+from typing import Any, Callable, Optional, Tuple, TypeVar, Union, cast
+
+from flask import Response, g
+
+CLIENT_AUTH: Optional[Union[Tuple[str, str], Any]] = None
+
+
+def init_app(_):
+    """Initializes authentication backend"""
+
+
+T = TypeVar("T", bound=Callable)
+
+
+def requires_authentication(function: T):
+    """Decorator for functions that require authentication"""
+
+    @wraps(function)
+    def decorated(*args, **kwargs):
+        if g.user.is_anonymous:
+            return Response("Unauthorized", 401, {})
+        return function(*args, **kwargs)
+
+    return cast(T, decorated)

airflow/api/client/__init__.py

@@ -27,14 +27,15 @@
 def get_current_api_client() -> Client:
     """Return current API Client based on current Airflow configuration"""
     api_module = import_module(conf.get('cli', 'api_client'))  # type: Any
-    auth_backend = api.load_auth()
+    auth_backends = api.load_auth()
     session = None
-    session_factory = getattr(auth_backend, 'create_client_session', None)
-    if session_factory:
-        session = session_factory()
-    api_client = api_module.Client(
-        api_base_url=conf.get('cli', 'endpoint_url'),
-        auth=getattr(auth_backend, 'CLIENT_AUTH', None),
-        session=session,
-    )
+    for backend in auth_backends:
+        session_factory = getattr(backend, 'create_client_session', None)
+        if session_factory:
+            session = session_factory()
+        api_client = api_module.Client(
+            api_base_url=conf.get('cli', 'endpoint_url'),
+            auth=getattr(backend, 'CLIENT_AUTH', None),
+            session=session,
+        )
     return api_client

airflow/api_connexion/openapi/v1.yaml

@@ -179,9 +179,9 @@ info:
     and it is even possible to add your own method.
 
     If you want to check which auth backend is currently set, you can use
-    `airflow config get-value api auth_backend` command as in the example below.
+    `airflow config get-value api auth_backends` command as in the example below.
     ```bash
-    $ airflow config get-value api auth_backend
+    $ airflow config get-value api auth_backends
     airflow.api.auth.backend.basic_auth
     ```
     The default is to deny all requests.

airflow/api_connexion/security.py

@@ -27,11 +27,13 @@
 
 def check_authentication() -> None:
     """Checks that the request has valid authorization information."""
-    response = current_app.api_auth.requires_authentication(Response)()
-    if response.status_code != 200:
-        # since this handler only checks authentication, not authorization,
-        # we should always return 401
-        raise Unauthenticated(headers=response.headers)
+    for auth in current_app.api_auth:
+        response = auth.requires_authentication(Response)()
+        if response.status_code == 200:
+            return
+    # since this handler only checks authentication, not authorization,
+    # we should always return 401
+    raise Unauthenticated(headers=response.headers)
 
 
 def requires_access(permissions: Optional[Sequence[Tuple[str, str]]] = None) -> Callable[[T], T]:

airflow/config_templates/config.yml

@@ -793,7 +793,7 @@
       type: boolean
       example: ~
       default: "False"
-    - name: auth_backend
+    - name: auth_backends
       description: |
         How to authenticate users of the API. See
         https://airflow.apache.org/docs/apache-airflow/stable/security.html for possible values.

airflow/config_templates/default_airflow.cfg

@@ -433,7 +433,7 @@ enable_experimental_api = False
 # How to authenticate users of the API. See
 # https://airflow.apache.org/docs/apache-airflow/stable/security.html for possible values.
 # ("airflow.api.auth.backend.default" allows all requests for historic reasons)
-auth_backend = airflow.api.auth.backend.deny_all
+auth_backends = airflow.api.auth.backend.deny_all
 
 # Used to set the maximum page limit for API requests
 maximum_page_limit = 100

airflow/config_templates/default_test.cfg

@@ -61,11 +61,12 @@ api_client = airflow.api.client.local_client
 endpoint_url = http://localhost:8080
 
 [api]
-auth_backend = airflow.api.auth.backend.default
+auth_backends = airflow.api.auth.backend.default
 
 [operators]
 default_owner = airflow
 
+
 [hive]
 default_hive_mapred_queue = airflow
 

airflow/configuration.py

@@ -185,6 +185,7 @@ class AirflowConfigParser(ConfigParser):
         ('core', 'max_active_tasks_per_dag'): ('core', 'dag_concurrency', '2.2.0'),
         ('logging', 'worker_log_server_port'): ('celery', 'worker_log_server_port', '2.2.0'),
         ('api', 'access_control_allow_origins'): ('api', 'access_control_allow_origin', '2.2.0'),
+        ('api', 'auth_backends'): ('api', 'auth_backend', '2.3'),
     }
 
     # A mapping of old default values that we want to change and warn the user

airflow/ui/README.md

@@ -35,7 +35,7 @@ To communicate with the API you need to adjust some environment variables for th
 Be sure to allow CORS headers and set up an auth backend on your Airflow instance.
 
 ```
-export AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.basic_auth
+export AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth
 export AIRFLOW__API__ACCESS_CONTROL_ALLOW_HEADERS=*
 export AIRFLOW__API__ACCESS_CONTROL_ALLOW_METHODS=*
 export AIRFLOW__API__ACCESS_CONTROL_ALLOW_ORIGIN=http://127.0.0.1:28080

airflow/www/api/experimental/endpoints.py

@@ -45,7 +45,8 @@ def requires_authentication(function: T):
 
     @wraps(function)
     def decorated(*args, **kwargs):
-        return current_app.api_auth.requires_authentication(function)(*args, **kwargs)
+        auth = current_app.api_auth[0]
+        return auth.requires_authentication(function)(*args, **kwargs)
 
     return cast(T, decorated)
 

airflow/www/extensions/init_jinja_globals.py

@@ -66,9 +66,13 @@ def prepare_jinja_globals():
             'airflow_version': airflow_version,
             'git_version': git_version,
             'k8s_or_k8scelery_executor': IS_K8S_OR_K8SCELERY_EXECUTOR,
-            'rest_api_enabled': conf.get('api', 'auth_backend') != 'airflow.api.auth.backend.deny_all',
+            'rest_api_enabled': False,
         }
 
+        backends = conf.get('api', 'auth_backends')
+        if len(backends) > 0 and backends[0] != 'airflow.api.auth.backend.deny_all':
+            extra_globals['rest_api_enabled'] = True
+
         if 'analytics_tool' in conf.getsection('webserver'):
             extra_globals.update(
                 {

airflow/www/extensions/init_security.py

@@ -43,16 +43,19 @@ def apply_caching(response):
 
 
 def init_api_experimental_auth(app):
-    """Loads authentication backend"""
-    auth_backend = 'airflow.api.auth.backend.default'
+    """Loads authentication backends"""
+    auth_backends = 'airflow.api.auth.backend.default'
     try:
-        auth_backend = conf.get("api", "auth_backend")
+        auth_backends = conf.get("api", "auth_backends")
     except AirflowConfigException:
         pass
 
-    try:
-        app.api_auth = import_module(auth_backend)
-        app.api_auth.init_app(app)
-    except ImportError as err:
-        log.critical("Cannot import %s for API authentication due to: %s", auth_backend, err)
-        raise AirflowException(err)
+    app.api_auth = []
+    for backend in auth_backends.split():
+        try:
+            auth = import_module(backend.strip())
+            auth.init_app(app)
+            app.api_auth.append(auth)
+        except ImportError as err:
+            log.critical("Cannot import %s for API authentication due to: %s", backend, err)
+            raise AirflowException(err)

airflow/www/static/js/connection_form.js

@@ -158,7 +158,7 @@ $(document).ready(() => {
   if (!restApiEnabled) {
     $(testConnBtn).prop('disabled', true)
       .attr('title', 'Airflow REST APIs have been disabled. '
-        + 'See api->auth_backend section of the Airflow configuration.');
+        + 'See api->auth_backends section of the Airflow configuration.');
   }
 
   $(testConnBtn).insertAfter($('form#model_form div.well.well-sm button:submit'));

chart/values.yaml

@@ -1384,7 +1384,7 @@ config:
     remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
   # Authentication backend used for the experimental API
   api:
-    auth_backend: airflow.api.auth.backend.deny_all
+    auth_backends: airflow.api.auth.backend.deny_all
   logging:
     remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
     colored_console_log: 'False'

docs/apache-airflow-providers-google/api-auth-backend/google-openid.rst

@@ -25,7 +25,7 @@ for authentication. To enable it, set the following option in the configuration:
 .. code-block:: ini
 
     [api]
-    auth_backend = airflow.providers.google.common.auth_backend.google_openid
+    auth_backends = airflow.providers.google.common.auth_backend.google_openid
 
 It is also highly recommended to configure an OAuth2 audience so that the generated tokens are restricted to
 use by Airflow only.

docs/apache-airflow/security/api.rst

@@ -27,19 +27,25 @@ deny all requests:
 .. code-block:: ini
 
     [api]
-    auth_backend = airflow.api.auth.backend.deny_all
+    auth_backends = airflow.api.auth.backend.deny_all
 
 .. versionchanged:: 1.10.11
 
     In Airflow <1.10.11, the default setting was to allow all API requests without authentication, but this
     posed security risks for if the Webserver is publicly accessible.
 
-If you want to check which authentication backend is currently set, you can use ``airflow config get-value api auth_backend``
+.. versionchanged:: 2.3
+
+    In Airflow <2.3 this setting was ``auth_backend`` and allowed only one
+    value. In 2.3 it was changed to support multiple backends that are tried
+    in turn.
+
+If you want to check which authentication backends are currently set, you can use ``airflow config get-value api auth_backends``
 command as in the example below.
 
 .. code-block:: console
 
-    $ airflow config get-value api auth_backend
+    $ airflow config get-value api auth_backends
     airflow.api.auth.backend.basic_auth
 
 Disable authentication
@@ -51,7 +57,7 @@ If you wish to have the experimental API work, and aware of the risks of enablin
 .. code-block:: ini
 
     [api]
-    auth_backend = airflow.api.auth.backend.default
+    auth_backends = airflow.api.auth.backend.default
 
 .. note::
 
@@ -69,7 +75,7 @@ To enable Kerberos authentication, set the following in the configuration:
 .. code-block:: ini
 
     [api]
-    auth_backend = airflow.api.auth.backend.kerberos_auth
+    auth_backends = airflow.api.auth.backend.kerberos_auth
 
     [kerberos]
     keytab = <KEYTAB>
@@ -89,7 +95,7 @@ To enable basic authentication, set the following in the configuration:
 .. code-block:: ini
 
     [api]
-    auth_backend = airflow.api.auth.backend.basic_auth
+    auth_backends = airflow.api.auth.backend.basic_auth
 
 Username and password needs to be base64 encoded and send through the
 ``Authorization`` HTTP header in the following format:
@@ -125,7 +131,7 @@ and may have one of the following to support API client authorizations used by :
 * function ``create_client_session() -> requests.Session``
 * attribute ``CLIENT_AUTH: Optional[Union[Tuple[str, str], requests.auth.AuthBase]]``
 
-After writing your backend module, provide the fully qualified module name in the ``auth_backend`` key in the ``[api]``
+After writing your backend module, provide the fully qualified module name in the ``auth_backends`` key in the ``[api]``
 section of ``airflow.cfg``.
 
 Additional options to your auth backend can be configured in ``airflow.cfg``, as a new option.

docs/apache-airflow/start/docker-compose.yaml

@@ -55,7 +55,7 @@ x-airflow-common:
     AIRFLOW__CORE__FERNET_KEY: ''
     AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
     AIRFLOW__CORE__LOAD_EXAMPLES: 'true'
-    AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
+    AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth'
     _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
   volumes:
     - ./dags:/opt/airflow/dags

docs/helm-chart/airflow-configuration.rst

@@ -46,7 +46,7 @@ application. See the bottom line of the example:
        remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
      # Authentication backend used for the experimental API
      api:
-       auth_backend: airflow.api.auth.backend.deny_all
+       auth_backends: airflow.api.auth.backend.deny_all
      logging:
        remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
        colored_console_log: 'False'

scripts/ci/libraries/_kind.sh

@@ -350,7 +350,7 @@ function kind::deploy_airflow_with_helm() {
         --set "images.airflow.repository=${AIRFLOW_IMAGE_KUBERNETES}" \
         --set "images.airflow.tag=latest" -v 1 \
         --set "defaultAirflowTag=latest" -v 1 \
-        --set "config.api.auth_backend=airflow.api.auth.backend.basic_auth" \
+        --set "config.api.auth_backends=airflow.api.auth.backend.basic_auth" \
         --set "config.logging.logging_level=DEBUG" \
         --set "executor=${EXECUTOR}"
     echo
@@ -383,7 +383,7 @@ function kind::upgrade_airflow_with_helm() {
         --set "images.airflow.repository=${AIRFLOW_IMAGE_KUBERNETES}" \
         --set "images.airflow.tag=latest" -v 1 \
         --set "defaultAirflowTag=latest" -v 1 \
-        --set "config.api.auth_backend=airflow.api.auth.backend.basic_auth" \
+        --set "config.api.auth_backends=airflow.api.auth.backend.basic_auth" \
         --set "config.logging.logging_level=DEBUG" \
         --set "executor=${mode}"
 

tests/api/auth/backend/test_kerberos_auth.py

@@ -37,7 +37,7 @@
 def app_for_kerberos():
     with conf_vars(
         {
-            ("api", "auth_backend"): "airflow.api.auth.backend.kerberos_auth",
+            ("api", "auth_backends"): "airflow.api.auth.backend.kerberos_auth",
             ("kerberos", "keytab"): KRB5_KTNAME,
             ('api', 'enable_experimental_api'): 'true',
         }