diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 2062b135f14eb..650989502c3ef 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -98,6 +98,10 @@ class AirflowFailException(AirflowException): """Raise when the task should be failed without retrying.""" +class AirflowOptionalProviderFeatureException(AirflowException): + """Raise by providers when imports are missing for optional provider features.""" + + class UnmappableXComTypePushed(AirflowException): """Raise when an unmappable type is pushed as a mapped downstream's dependency.""" diff --git a/airflow/hooks/base.py b/airflow/hooks/base.py index 3531bf3800d0b..5a1e01e6fb4d2 100644 --- a/airflow/hooks/base.py +++ b/airflow/hooks/base.py @@ -97,6 +97,14 @@ def get_conn(self) -> Any: """Returns connection for the hook.""" raise NotImplementedError() + @classmethod + def get_connection_form_widgets(cls) -> Dict[str, Any]: + ... + + @classmethod + def get_ui_field_behaviour(cls) -> Dict[str, Any]: + ... + class DiscoverableHook(Protocol): """ @@ -159,7 +167,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: ... @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """ Returns dictionary describing customizations to implement in javascript handling the connection form. Should be compliant with airflow/customized_form_field_behaviours.schema.json' diff --git a/airflow/providers/apache/spark/hooks/spark_submit.py b/airflow/providers/apache/spark/hooks/spark_submit.py index 95a65c853df00..ecfb05b37a152 100644 --- a/airflow/providers/apache/spark/hooks/spark_submit.py +++ b/airflow/providers/apache/spark/hooks/spark_submit.py @@ -86,7 +86,7 @@ class SparkSubmitHook(BaseHook, LoggingMixin): hook_name = 'Spark' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'login', 'password'], diff --git a/airflow/providers/asana/hooks/asana.py b/airflow/providers/asana/hooks/asana.py index 6e633751268f5..8c9f2e045a64b 100644 --- a/airflow/providers/asana/hooks/asana.py +++ b/airflow/providers/asana/hooks/asana.py @@ -62,7 +62,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ["port", "host", "login", "schema"], diff --git a/airflow/providers/cloudant/hooks/cloudant.py b/airflow/providers/cloudant/hooks/cloudant.py index 3c02a945fd1d3..bce3456e36333 100644 --- a/airflow/providers/cloudant/hooks/cloudant.py +++ b/airflow/providers/cloudant/hooks/cloudant.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """Hook for Cloudant""" -from typing import Dict +from typing import Any, Dict from cloudant import cloudant @@ -39,7 +39,7 @@ class CloudantHook(BaseHook): hook_name = 'Cloudant' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['port', 'extra'], diff --git a/airflow/providers/cncf/kubernetes/hooks/kubernetes.py b/airflow/providers/cncf/kubernetes/hooks/kubernetes.py index 8025c2378105c..38305031dd022 100644 --- a/airflow/providers/cncf/kubernetes/hooks/kubernetes.py +++ b/airflow/providers/cncf/kubernetes/hooks/kubernetes.py @@ -92,7 +92,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['host', 'schema', 'login', 'password', 'port', 'extra'], diff --git a/airflow/providers/docker/hooks/docker.py b/airflow/providers/docker/hooks/docker.py index 7ac2122d9e8fe..e497140586fbb 100644 --- a/airflow/providers/docker/hooks/docker.py +++ b/airflow/providers/docker/hooks/docker.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Dict, Optional +from typing import Any, Dict, Optional from docker import APIClient from docker.errors import APIError @@ -39,7 +39,7 @@ class DockerHook(BaseHook, LoggingMixin): hook_name = 'Docker' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema'], diff --git a/airflow/providers/google/cloud/hooks/compute_ssh.py b/airflow/providers/google/cloud/hooks/compute_ssh.py index 9a930d5fa36d2..ff4640eba9dc3 100644 --- a/airflow/providers/google/cloud/hooks/compute_ssh.py +++ b/airflow/providers/google/cloud/hooks/compute_ssh.py @@ -90,7 +90,7 @@ class ComputeEngineSSHHook(SSHHook): hook_name = 'Google Cloud SSH' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: return { "hidden_fields": ['host', 'schema', 'login', 'password', 'port', 'extra'], "relabeling": {}, diff --git a/airflow/providers/google/common/hooks/base_google.py b/airflow/providers/google/common/hooks/base_google.py index a9ee05c1400ff..b2dd3f9937737 100644 --- a/airflow/providers/google/common/hooks/base_google.py +++ b/airflow/providers/google/common/hooks/base_google.py @@ -197,7 +197,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['host', 'schema', 'login', 'password', 'port', 'extra'], diff --git a/airflow/providers/google/leveldb/hooks/leveldb.py b/airflow/providers/google/leveldb/hooks/leveldb.py index 2ca78f01f9a85..bff3f6af9072e 100644 --- a/airflow/providers/google/leveldb/hooks/leveldb.py +++ b/airflow/providers/google/leveldb/hooks/leveldb.py @@ -17,11 +17,24 @@ """Hook for Level DB""" from typing import List, Optional -import plyvel -from plyvel import DB - -from airflow.exceptions import AirflowException -from airflow.hooks.base import BaseHook +try: + import plyvel + from plyvel import DB + + from airflow.exceptions import AirflowException + from airflow.hooks.base import BaseHook + +except ImportError as e: + # Plyvel is an optional feature and if imports are missing, it should be silently ignored + # As of Airflow 2.3 and above the operator can throw OptionalProviderFeatureException + try: + from airflow.exceptions import AirflowOptionalProviderFeatureException + except ImportError: + # However, in order to keep backwards-compatibility with Airflow 2.1 and 2.2, if the + # 2.3 exception cannot be imported, the original ImportError should be raised. + # This try/except can be removed when the provider depends on Airflow >= 2.3.0 + raise e from None + raise AirflowOptionalProviderFeatureException(e) DB_NOT_INITIALIZED_BEFORE = "The `get_conn` method should be called before!" diff --git a/airflow/providers/jdbc/hooks/jdbc.py b/airflow/providers/jdbc/hooks/jdbc.py index 54a3345e6ed5e..734afecb5ba18 100644 --- a/airflow/providers/jdbc/hooks/jdbc.py +++ b/airflow/providers/jdbc/hooks/jdbc.py @@ -54,7 +54,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['port', 'schema', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/adx.py b/airflow/providers/microsoft/azure/hooks/adx.py index 6e8172fa3ae98..750fa051b2dc4 100644 --- a/airflow/providers/microsoft/azure/hooks/adx.py +++ b/airflow/providers/microsoft/azure/hooks/adx.py @@ -97,7 +97,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/base_azure.py b/airflow/providers/microsoft/azure/hooks/base_azure.py index c0ba06bf88672..3205809dd4b5d 100644 --- a/airflow/providers/microsoft/azure/hooks/base_azure.py +++ b/airflow/providers/microsoft/azure/hooks/base_azure.py @@ -57,7 +57,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" import json diff --git a/airflow/providers/microsoft/azure/hooks/batch.py b/airflow/providers/microsoft/azure/hooks/batch.py index e2ea258061ab3..7e1f0ac5bb749 100644 --- a/airflow/providers/microsoft/azure/hooks/batch.py +++ b/airflow/providers/microsoft/azure/hooks/batch.py @@ -56,7 +56,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/container_registry.py b/airflow/providers/microsoft/azure/hooks/container_registry.py index f02fc795f0ab6..6cc8e985178b3 100644 --- a/airflow/providers/microsoft/azure/hooks/container_registry.py +++ b/airflow/providers/microsoft/azure/hooks/container_registry.py @@ -17,7 +17,7 @@ # under the License. """Hook for Azure Container Registry""" -from typing import Dict +from typing import Any, Dict from azure.mgmt.containerinstance.models import ImageRegistryCredential @@ -39,7 +39,7 @@ class AzureContainerRegistryHook(BaseHook): hook_name = 'Azure Container Registry' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/container_volume.py b/airflow/providers/microsoft/azure/hooks/container_volume.py index 4054a47a8ac87..fbd0e18723d6a 100644 --- a/airflow/providers/microsoft/azure/hooks/container_volume.py +++ b/airflow/providers/microsoft/azure/hooks/container_volume.py @@ -54,7 +54,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', "extra"], diff --git a/airflow/providers/microsoft/azure/hooks/cosmos.py b/airflow/providers/microsoft/azure/hooks/cosmos.py index f54b81a6971c2..007273a3957c2 100644 --- a/airflow/providers/microsoft/azure/hooks/cosmos.py +++ b/airflow/providers/microsoft/azure/hooks/cosmos.py @@ -67,7 +67,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/data_factory.py b/airflow/providers/microsoft/azure/hooks/data_factory.py index 2781258d3a9c4..8e1e97d2a3fbe 100644 --- a/airflow/providers/microsoft/azure/hooks/data_factory.py +++ b/airflow/providers/microsoft/azure/hooks/data_factory.py @@ -144,7 +144,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/data_lake.py b/airflow/providers/microsoft/azure/hooks/data_lake.py index bb293805cdbf5..f386947041a55 100644 --- a/airflow/providers/microsoft/azure/hooks/data_lake.py +++ b/airflow/providers/microsoft/azure/hooks/data_lake.py @@ -65,7 +65,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/fileshare.py b/airflow/providers/microsoft/azure/hooks/fileshare.py index 0157c2a788218..1f78d6e4b2084 100644 --- a/airflow/providers/microsoft/azure/hooks/fileshare.py +++ b/airflow/providers/microsoft/azure/hooks/fileshare.py @@ -64,7 +64,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 9f7ce5a19f1a2..22089407ff857 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -81,7 +81,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port'], diff --git a/airflow/providers/pagerduty/hooks/pagerduty.py b/airflow/providers/pagerduty/hooks/pagerduty.py index b921e5c42f9af..98be80965aaab 100644 --- a/airflow/providers/pagerduty/hooks/pagerduty.py +++ b/airflow/providers/pagerduty/hooks/pagerduty.py @@ -49,7 +49,7 @@ class PagerdutyHook(BaseHook): hook_name = "Pagerduty" @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['port', 'login', 'schema', 'host'], diff --git a/airflow/providers/pagerduty/hooks/pagerduty_events.py b/airflow/providers/pagerduty/hooks/pagerduty_events.py index 08e6265ae77fb..c5eaffe105284 100644 --- a/airflow/providers/pagerduty/hooks/pagerduty_events.py +++ b/airflow/providers/pagerduty/hooks/pagerduty_events.py @@ -41,7 +41,7 @@ class PagerdutyEventsHook(BaseHook): hook_name = "Pagerduty Events" @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['port', 'login', 'schema', 'host', 'extra'], diff --git a/airflow/providers/qubole/hooks/qubole.py b/airflow/providers/qubole/hooks/qubole.py index 7f12fb446d1f9..a62243e9a037d 100644 --- a/airflow/providers/qubole/hooks/qubole.py +++ b/airflow/providers/qubole/hooks/qubole.py @@ -22,7 +22,7 @@ import os import pathlib import time -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from qds_sdk.commands import ( Command, @@ -119,7 +119,7 @@ class QuboleHook(BaseHook): hook_name = 'Qubole' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['login', 'schema', 'port', 'extra'], diff --git a/airflow/providers/salesforce/hooks/salesforce.py b/airflow/providers/salesforce/hooks/salesforce.py index da928421cbf0b..cdb2c918e935c 100644 --- a/airflow/providers/salesforce/hooks/salesforce.py +++ b/airflow/providers/salesforce/hooks/salesforce.py @@ -119,7 +119,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ["schema", "port", "extra", "host"], diff --git a/airflow/providers/sftp/hooks/sftp.py b/airflow/providers/sftp/hooks/sftp.py index 23afa6c2d0dc5..49b063f9b26ad 100644 --- a/airflow/providers/sftp/hooks/sftp.py +++ b/airflow/providers/sftp/hooks/sftp.py @@ -19,7 +19,7 @@ import datetime import stat import warnings -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import pysftp import tenacity @@ -61,7 +61,7 @@ class SFTPHook(SSHHook): hook_name = 'SFTP' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: return { "hidden_fields": ['schema'], "relabeling": { diff --git a/airflow/providers/snowflake/hooks/snowflake.py b/airflow/providers/snowflake/hooks/snowflake.py index a61f6e0af13ce..9fa13f67a049f 100644 --- a/airflow/providers/snowflake/hooks/snowflake.py +++ b/airflow/providers/snowflake/hooks/snowflake.py @@ -97,7 +97,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" import json diff --git a/airflow/providers/ssh/hooks/ssh.py b/airflow/providers/ssh/hooks/ssh.py index db98aad42f519..39b64aac787d8 100644 --- a/airflow/providers/ssh/hooks/ssh.py +++ b/airflow/providers/ssh/hooks/ssh.py @@ -88,7 +88,7 @@ class SSHHook(BaseHook): hook_name = 'SSH' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema'], diff --git a/airflow/providers/yandex/hooks/yandex.py b/airflow/providers/yandex/hooks/yandex.py index 84cc3ee45641b..a337954496241 100644 --- a/airflow/providers/yandex/hooks/yandex.py +++ b/airflow/providers/yandex/hooks/yandex.py @@ -95,7 +95,7 @@ def provider_user_agent(cls) -> Optional[str]: return None @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['host', 'schema', 'login', 'password', 'port', 'extra'], diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index b5b84aa21e8df..a36b592d37d23 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -18,7 +18,6 @@ """Manages all providers.""" import fnmatch import functools -import importlib import json import logging import os @@ -27,11 +26,26 @@ from collections import OrderedDict from functools import wraps from time import perf_counter -from typing import Any, Callable, Dict, List, MutableMapping, NamedTuple, Optional, Set, TypeVar, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + MutableMapping, + NamedTuple, + Optional, + Set, + Type, + TypeVar, + Union, + cast, +) import jsonschema from packaging import version as packaging_version +from airflow.exceptions import AirflowOptionalProviderFeatureException +from airflow.hooks.base import BaseHook from airflow.utils import yaml from airflow.utils.entry_points import entry_points_with_dist from airflow.utils.log.logging_mixin import LoggingMixin @@ -124,7 +138,7 @@ def _check_builtin_provider_prefix(provider_package: str, class_name: str) -> bo return True -def _sanity_check(provider_package: str, class_name: str) -> bool: +def _sanity_check(provider_package: str, class_name: str) -> Optional[Type[BaseHook]]: """ Performs coherence check on provider classes. For apache-airflow providers - it checks if it starts with appropriate package. For all providers @@ -134,31 +148,42 @@ def _sanity_check(provider_package: str, class_name: str) -> bool: :param provider_package: name of the provider package :param class_name: name of the class to import - :return True if the class is OK, False otherwise. + :return the class if the class is OK, None otherwise. """ if not _check_builtin_provider_prefix(provider_package, class_name): - return False + return None try: - import_string(class_name) + imported_class = import_string(class_name) + except AirflowOptionalProviderFeatureException as e: + # When the provider class raises AirflowOptionalProviderFeatureException + # this is an expected case when only some classes in provider are + # available. We just log debug level here + log.debug( + "Optional feature disabled on exception when importing '%s' from '%s' package", + class_name, + provider_package, + exc_info=e, + ) + return None except ImportError as e: # When there is an ImportError we turn it into debug warnings as this is # an expected case when only some providers are installed - log.debug( - "Exception when importing '%s' from '%s' package: %s", + log.warning( + "Exception when importing '%s' from '%s' package", class_name, provider_package, - e, + exc_info=e, ) - return False + return None except Exception as e: log.warning( - "Exception when importing '%s' from '%s' package: %s", + "Exception when importing '%s' from '%s' package", class_name, provider_package, - e, + exc_info=e, ) - return False - return True + return None + return imported_class class ProviderInfo(NamedTuple): @@ -631,11 +656,11 @@ def _import_hook( f"Provider package name is not set when hook_class_name ({hook_class_name}) is used" ) allowed_field_classes = [IntegerField, PasswordField, StringField, BooleanField] - if not _sanity_check(package_name, hook_class_name): + hook_class = _sanity_check(package_name, hook_class_name) + if hook_class is None: return None try: module, class_name = hook_class_name.rsplit('.', maxsplit=1) - hook_class = getattr(importlib.import_module(module), class_name) # Do not use attr here. We want to check only direct class fields not those # inherited from parent hook. This way we add form fields only once for the whole # hierarchy and we add it only from the parent hook that provides those! diff --git a/docs/apache-airflow-providers/howto/create-update-providers.rst b/docs/apache-airflow-providers/howto/create-update-providers.rst index 5ad4c65912969..f953f46195ea0 100644 --- a/docs/apache-airflow-providers/howto/create-update-providers.rst +++ b/docs/apache-airflow-providers/howto/create-update-providers.rst @@ -304,6 +304,53 @@ main Airflow documentation that involves some steps with the providers is also w ./breeze build-docs -- --package-filter apache-airflow-providers- ./breeze build-docs -- --package-filter apache-airflow +Optional provider features +-------------------------- + + .. note:: + + This feature is available in Airflow 2.3+. + +Some providers might provide optional features, which are only available when some packages or libraries +are installed. Such features will typically result in ``ImportErrors`` however those import errors +should be silently ignored rather than pollute the logs of Airflow with false warnings. False warnings +are a very bad pattern, as they tend to turn into blind spots, so avoiding false warnings is encouraged. +However until Airflow 2.3, Airflow had no mechanism to selectively ignore "known" ImportErrors. So +Airflow 2.1 and 2.2 silently ignored all ImportErrors coming from providers with actually lead to +ignoring even important import errors - without giving the clue to Airflow users that there is something +missing in provider dependencies. + +In Airflow 2.3, new exception :class:`~airflow.exceptions.OptionalProviderFeatureException` has been +introduced and Providers can use the exception to signal that the ImportError (or any other error) should +be ignored by Airflow ProvidersManager. However this Exception is only available in Airflow 2.3 so if +providers would like to remain compatible with Airflow 2.1 and 2.2, they should continue throwing +the ImportError exception. + +Example code (from Plyvel Hook, part of the Google Provider) explains how such conditional error handling +should be implemented to keep compatibility with Airflow 2.1 and 2.2 + + .. code-block:: python + + try: + import plyvel + from plyvel import DB + + from airflow.exceptions import AirflowException + from airflow.hooks.base import BaseHook + + except ImportError as e: + # Plyvel is an optional feature and if imports are missing, it should be silently ignored + # As of Airflow 2.3 and above the operator can throw OptionalProviderFeatureException + try: + from airflow.exceptions import AirflowOptionalProviderFeatureException + except ImportError: + # However, in order to keep backwards-compatibility with Airflow 2.1 and 2.2, if the + # 2.3 exception cannot be imported, the original ImportError should be raised. + # This try/except can be removed when the provider depends on Airflow >= 2.3.0 + raise e + raise AirflowOptionalProviderFeatureException(e) + + How-to Update a community provider ---------------------------------- diff --git a/tests/core/test_providers_manager.py b/tests/core/test_providers_manager.py index 6f7b709356dd4..b38ec8d3b5661 100644 --- a/tests/core/test_providers_manager.py +++ b/tests/core/test_providers_manager.py @@ -17,14 +17,13 @@ # under the License. import logging import re -import sys import unittest -from unittest.mock import call, patch +from unittest.mock import patch import pytest -import airflow -from airflow.providers_manager import ProviderInfo, ProvidersManager +from airflow.exceptions import AirflowOptionalProviderFeatureException +from airflow.providers_manager import HookClassProvider, ProviderInfo, ProvidersManager class TestProviderManager(unittest.TestCase): @@ -116,13 +115,7 @@ def test_warning_logs_not_generated(self): assert not self._caplog.records assert "sftp" in providers_manager.hooks - @patch('airflow.providers_manager.importlib.import_module') - def test_hooks(self, mock_import_module): - # importlib.resources relies on importing the containing module to read - # its content. The provider manager needs to read two validation schemas - # 'provider_info.schema.json' and 'customized_form_field_behaviours.schema.json'. - mock_import_module.side_effect = [airflow, airflow] - + def test_hooks(self): with pytest.warns(expected_warning=None) as warning_records: with self._caplog.at_level(logging.WARNING): provider_manager = ProvidersManager() @@ -131,16 +124,6 @@ def test_hooks(self, mock_import_module): assert [] == [w.message for w in warning_records.list if "hook-class-names" in str(w.message)] assert len(self._caplog.records) == 0 - # The stdlib importlib.resources implementation in 3.9 does not rely on - # importlib.import_module, so the function should not be called. The - # implementation for 3.10+ and the backport we use for up to 3.8 uses it - # to import the top-level 'airflow' for the two schema files. Also see - # comment on mocking at the beginning of the function. - if sys.version_info[:2] == (3, 9): - assert mock_import_module.mock_calls == [] - else: - assert mock_import_module.mock_calls == [call("airflow"), call("airflow")] - def test_hook_values(self): with pytest.warns(expected_warning=None) as warning_records: with self._caplog.at_level(logging.WARNING): @@ -179,3 +162,32 @@ def test_auth_backends(self): provider_manager = ProvidersManager() auth_backend_module_names = list(provider_manager.auth_backend_module_names) assert len(auth_backend_module_names) > 0 + + @patch("airflow.providers_manager.import_string") + def test_optional_feature_no_warning(self, mock_importlib_import_string): + with self._caplog.at_level(logging.WARNING): + mock_importlib_import_string.side_effect = AirflowOptionalProviderFeatureException() + providers_manager = ProvidersManager() + providers_manager._hook_provider_dict["test_connection"] = HookClassProvider( + package_name="test_package", hook_class_name="HookClass" + ) + providers_manager._import_hook( + hook_class_name=None, package_name=None, connection_type="test_connection" + ) + assert [] == self._caplog.messages + + @patch("airflow.providers_manager.import_string") + def test_optional_feature_debug(self, mock_importlib_import_string): + with self._caplog.at_level(logging.DEBUG): + mock_importlib_import_string.side_effect = AirflowOptionalProviderFeatureException() + providers_manager = ProvidersManager() + providers_manager._hook_provider_dict["test_connection"] = HookClassProvider( + package_name="test_package", hook_class_name="HookClass" + ) + providers_manager._import_hook( + hook_class_name=None, package_name=None, connection_type="test_connection" + ) + assert [ + "Optional feature disabled on exception when importing 'HookClass' from " + "'test_package' package" + ] == self._caplog.messages