From cb73053211367e2c2dd76d5279cdc7dc7b190124 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 27 Jan 2022 13:58:53 +0100 Subject: [PATCH] Add optional features in providers. (#21074) Some features in providers can be optional, depending on the presence of some libraries. Since Providers Manager tries to import the right classes that are exposed via providers it should not - in this case - log warning message for those optional features. Previously, all ImportErrors were turned into debug log but now we only turn them in debug log when creator of the provider deliberately raised an AirflowOptionalProviderFeatureException. Instructions on how to raise such exception in the way to keep backwards compatibility were updated in proider's documentation. Fixes: #20709 --- airflow/exceptions.py | 4 ++ airflow/hooks/base.py | 10 +++- .../apache/spark/hooks/spark_submit.py | 2 +- airflow/providers/asana/hooks/asana.py | 2 +- airflow/providers/cloudant/hooks/cloudant.py | 4 +- .../cncf/kubernetes/hooks/kubernetes.py | 2 +- airflow/providers/docker/hooks/docker.py | 4 +- .../google/cloud/hooks/compute_ssh.py | 2 +- .../google/common/hooks/base_google.py | 2 +- .../providers/google/leveldb/hooks/leveldb.py | 23 ++++++-- airflow/providers/jdbc/hooks/jdbc.py | 2 +- .../providers/microsoft/azure/hooks/adx.py | 2 +- .../microsoft/azure/hooks/base_azure.py | 2 +- .../providers/microsoft/azure/hooks/batch.py | 2 +- .../azure/hooks/container_registry.py | 4 +- .../microsoft/azure/hooks/container_volume.py | 2 +- .../providers/microsoft/azure/hooks/cosmos.py | 2 +- .../microsoft/azure/hooks/data_factory.py | 2 +- .../microsoft/azure/hooks/data_lake.py | 2 +- .../microsoft/azure/hooks/fileshare.py | 2 +- .../providers/microsoft/azure/hooks/wasb.py | 2 +- .../providers/pagerduty/hooks/pagerduty.py | 2 +- .../pagerduty/hooks/pagerduty_events.py | 2 +- airflow/providers/qubole/hooks/qubole.py | 4 +- .../providers/salesforce/hooks/salesforce.py | 2 +- airflow/providers/sftp/hooks/sftp.py | 4 +- .../providers/snowflake/hooks/snowflake.py | 2 +- airflow/providers/ssh/hooks/ssh.py | 2 +- airflow/providers/yandex/hooks/yandex.py | 2 +- airflow/providers_manager.py | 57 +++++++++++++------ .../howto/create-update-providers.rst | 47 +++++++++++++++ tests/core/test_providers_manager.py | 54 +++++++++++------- 32 files changed, 183 insertions(+), 74 deletions(-) diff --git a/airflow/exceptions.py b/airflow/exceptions.py index 2062b135f14eb..650989502c3ef 100644 --- a/airflow/exceptions.py +++ b/airflow/exceptions.py @@ -98,6 +98,10 @@ class AirflowFailException(AirflowException): """Raise when the task should be failed without retrying.""" +class AirflowOptionalProviderFeatureException(AirflowException): + """Raise by providers when imports are missing for optional provider features.""" + + class UnmappableXComTypePushed(AirflowException): """Raise when an unmappable type is pushed as a mapped downstream's dependency.""" diff --git a/airflow/hooks/base.py b/airflow/hooks/base.py index 3531bf3800d0b..5a1e01e6fb4d2 100644 --- a/airflow/hooks/base.py +++ b/airflow/hooks/base.py @@ -97,6 +97,14 @@ def get_conn(self) -> Any: """Returns connection for the hook.""" raise NotImplementedError() + @classmethod + def get_connection_form_widgets(cls) -> Dict[str, Any]: + ... + + @classmethod + def get_ui_field_behaviour(cls) -> Dict[str, Any]: + ... + class DiscoverableHook(Protocol): """ @@ -159,7 +167,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: ... @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """ Returns dictionary describing customizations to implement in javascript handling the connection form. Should be compliant with airflow/customized_form_field_behaviours.schema.json' diff --git a/airflow/providers/apache/spark/hooks/spark_submit.py b/airflow/providers/apache/spark/hooks/spark_submit.py index 95a65c853df00..ecfb05b37a152 100644 --- a/airflow/providers/apache/spark/hooks/spark_submit.py +++ b/airflow/providers/apache/spark/hooks/spark_submit.py @@ -86,7 +86,7 @@ class SparkSubmitHook(BaseHook, LoggingMixin): hook_name = 'Spark' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'login', 'password'], diff --git a/airflow/providers/asana/hooks/asana.py b/airflow/providers/asana/hooks/asana.py index 6e633751268f5..8c9f2e045a64b 100644 --- a/airflow/providers/asana/hooks/asana.py +++ b/airflow/providers/asana/hooks/asana.py @@ -62,7 +62,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ["port", "host", "login", "schema"], diff --git a/airflow/providers/cloudant/hooks/cloudant.py b/airflow/providers/cloudant/hooks/cloudant.py index 3c02a945fd1d3..bce3456e36333 100644 --- a/airflow/providers/cloudant/hooks/cloudant.py +++ b/airflow/providers/cloudant/hooks/cloudant.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. """Hook for Cloudant""" -from typing import Dict +from typing import Any, Dict from cloudant import cloudant @@ -39,7 +39,7 @@ class CloudantHook(BaseHook): hook_name = 'Cloudant' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['port', 'extra'], diff --git a/airflow/providers/cncf/kubernetes/hooks/kubernetes.py b/airflow/providers/cncf/kubernetes/hooks/kubernetes.py index 8025c2378105c..38305031dd022 100644 --- a/airflow/providers/cncf/kubernetes/hooks/kubernetes.py +++ b/airflow/providers/cncf/kubernetes/hooks/kubernetes.py @@ -92,7 +92,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['host', 'schema', 'login', 'password', 'port', 'extra'], diff --git a/airflow/providers/docker/hooks/docker.py b/airflow/providers/docker/hooks/docker.py index 7ac2122d9e8fe..e497140586fbb 100644 --- a/airflow/providers/docker/hooks/docker.py +++ b/airflow/providers/docker/hooks/docker.py @@ -15,7 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Dict, Optional +from typing import Any, Dict, Optional from docker import APIClient from docker.errors import APIError @@ -39,7 +39,7 @@ class DockerHook(BaseHook, LoggingMixin): hook_name = 'Docker' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema'], diff --git a/airflow/providers/google/cloud/hooks/compute_ssh.py b/airflow/providers/google/cloud/hooks/compute_ssh.py index 9a930d5fa36d2..ff4640eba9dc3 100644 --- a/airflow/providers/google/cloud/hooks/compute_ssh.py +++ b/airflow/providers/google/cloud/hooks/compute_ssh.py @@ -90,7 +90,7 @@ class ComputeEngineSSHHook(SSHHook): hook_name = 'Google Cloud SSH' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: return { "hidden_fields": ['host', 'schema', 'login', 'password', 'port', 'extra'], "relabeling": {}, diff --git a/airflow/providers/google/common/hooks/base_google.py b/airflow/providers/google/common/hooks/base_google.py index a9ee05c1400ff..b2dd3f9937737 100644 --- a/airflow/providers/google/common/hooks/base_google.py +++ b/airflow/providers/google/common/hooks/base_google.py @@ -197,7 +197,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['host', 'schema', 'login', 'password', 'port', 'extra'], diff --git a/airflow/providers/google/leveldb/hooks/leveldb.py b/airflow/providers/google/leveldb/hooks/leveldb.py index 2ca78f01f9a85..bff3f6af9072e 100644 --- a/airflow/providers/google/leveldb/hooks/leveldb.py +++ b/airflow/providers/google/leveldb/hooks/leveldb.py @@ -17,11 +17,24 @@ """Hook for Level DB""" from typing import List, Optional -import plyvel -from plyvel import DB - -from airflow.exceptions import AirflowException -from airflow.hooks.base import BaseHook +try: + import plyvel + from plyvel import DB + + from airflow.exceptions import AirflowException + from airflow.hooks.base import BaseHook + +except ImportError as e: + # Plyvel is an optional feature and if imports are missing, it should be silently ignored + # As of Airflow 2.3 and above the operator can throw OptionalProviderFeatureException + try: + from airflow.exceptions import AirflowOptionalProviderFeatureException + except ImportError: + # However, in order to keep backwards-compatibility with Airflow 2.1 and 2.2, if the + # 2.3 exception cannot be imported, the original ImportError should be raised. + # This try/except can be removed when the provider depends on Airflow >= 2.3.0 + raise e from None + raise AirflowOptionalProviderFeatureException(e) DB_NOT_INITIALIZED_BEFORE = "The `get_conn` method should be called before!" diff --git a/airflow/providers/jdbc/hooks/jdbc.py b/airflow/providers/jdbc/hooks/jdbc.py index 54a3345e6ed5e..734afecb5ba18 100644 --- a/airflow/providers/jdbc/hooks/jdbc.py +++ b/airflow/providers/jdbc/hooks/jdbc.py @@ -54,7 +54,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['port', 'schema', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/adx.py b/airflow/providers/microsoft/azure/hooks/adx.py index 6e8172fa3ae98..750fa051b2dc4 100644 --- a/airflow/providers/microsoft/azure/hooks/adx.py +++ b/airflow/providers/microsoft/azure/hooks/adx.py @@ -97,7 +97,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/base_azure.py b/airflow/providers/microsoft/azure/hooks/base_azure.py index c0ba06bf88672..3205809dd4b5d 100644 --- a/airflow/providers/microsoft/azure/hooks/base_azure.py +++ b/airflow/providers/microsoft/azure/hooks/base_azure.py @@ -57,7 +57,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" import json diff --git a/airflow/providers/microsoft/azure/hooks/batch.py b/airflow/providers/microsoft/azure/hooks/batch.py index e2ea258061ab3..7e1f0ac5bb749 100644 --- a/airflow/providers/microsoft/azure/hooks/batch.py +++ b/airflow/providers/microsoft/azure/hooks/batch.py @@ -56,7 +56,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/container_registry.py b/airflow/providers/microsoft/azure/hooks/container_registry.py index f02fc795f0ab6..6cc8e985178b3 100644 --- a/airflow/providers/microsoft/azure/hooks/container_registry.py +++ b/airflow/providers/microsoft/azure/hooks/container_registry.py @@ -17,7 +17,7 @@ # under the License. """Hook for Azure Container Registry""" -from typing import Dict +from typing import Any, Dict from azure.mgmt.containerinstance.models import ImageRegistryCredential @@ -39,7 +39,7 @@ class AzureContainerRegistryHook(BaseHook): hook_name = 'Azure Container Registry' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/container_volume.py b/airflow/providers/microsoft/azure/hooks/container_volume.py index 4054a47a8ac87..fbd0e18723d6a 100644 --- a/airflow/providers/microsoft/azure/hooks/container_volume.py +++ b/airflow/providers/microsoft/azure/hooks/container_volume.py @@ -54,7 +54,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', "extra"], diff --git a/airflow/providers/microsoft/azure/hooks/cosmos.py b/airflow/providers/microsoft/azure/hooks/cosmos.py index f54b81a6971c2..007273a3957c2 100644 --- a/airflow/providers/microsoft/azure/hooks/cosmos.py +++ b/airflow/providers/microsoft/azure/hooks/cosmos.py @@ -67,7 +67,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/data_factory.py b/airflow/providers/microsoft/azure/hooks/data_factory.py index 2781258d3a9c4..8e1e97d2a3fbe 100644 --- a/airflow/providers/microsoft/azure/hooks/data_factory.py +++ b/airflow/providers/microsoft/azure/hooks/data_factory.py @@ -144,7 +144,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/data_lake.py b/airflow/providers/microsoft/azure/hooks/data_lake.py index bb293805cdbf5..f386947041a55 100644 --- a/airflow/providers/microsoft/azure/hooks/data_lake.py +++ b/airflow/providers/microsoft/azure/hooks/data_lake.py @@ -65,7 +65,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/fileshare.py b/airflow/providers/microsoft/azure/hooks/fileshare.py index 0157c2a788218..1f78d6e4b2084 100644 --- a/airflow/providers/microsoft/azure/hooks/fileshare.py +++ b/airflow/providers/microsoft/azure/hooks/fileshare.py @@ -64,7 +64,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port', 'host', 'extra'], diff --git a/airflow/providers/microsoft/azure/hooks/wasb.py b/airflow/providers/microsoft/azure/hooks/wasb.py index 9f7ce5a19f1a2..22089407ff857 100644 --- a/airflow/providers/microsoft/azure/hooks/wasb.py +++ b/airflow/providers/microsoft/azure/hooks/wasb.py @@ -81,7 +81,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema', 'port'], diff --git a/airflow/providers/pagerduty/hooks/pagerduty.py b/airflow/providers/pagerduty/hooks/pagerduty.py index b921e5c42f9af..98be80965aaab 100644 --- a/airflow/providers/pagerduty/hooks/pagerduty.py +++ b/airflow/providers/pagerduty/hooks/pagerduty.py @@ -49,7 +49,7 @@ class PagerdutyHook(BaseHook): hook_name = "Pagerduty" @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['port', 'login', 'schema', 'host'], diff --git a/airflow/providers/pagerduty/hooks/pagerduty_events.py b/airflow/providers/pagerduty/hooks/pagerduty_events.py index 08e6265ae77fb..c5eaffe105284 100644 --- a/airflow/providers/pagerduty/hooks/pagerduty_events.py +++ b/airflow/providers/pagerduty/hooks/pagerduty_events.py @@ -41,7 +41,7 @@ class PagerdutyEventsHook(BaseHook): hook_name = "Pagerduty Events" @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['port', 'login', 'schema', 'host', 'extra'], diff --git a/airflow/providers/qubole/hooks/qubole.py b/airflow/providers/qubole/hooks/qubole.py index 7f12fb446d1f9..a62243e9a037d 100644 --- a/airflow/providers/qubole/hooks/qubole.py +++ b/airflow/providers/qubole/hooks/qubole.py @@ -22,7 +22,7 @@ import os import pathlib import time -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from qds_sdk.commands import ( Command, @@ -119,7 +119,7 @@ class QuboleHook(BaseHook): hook_name = 'Qubole' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['login', 'schema', 'port', 'extra'], diff --git a/airflow/providers/salesforce/hooks/salesforce.py b/airflow/providers/salesforce/hooks/salesforce.py index da928421cbf0b..cdb2c918e935c 100644 --- a/airflow/providers/salesforce/hooks/salesforce.py +++ b/airflow/providers/salesforce/hooks/salesforce.py @@ -119,7 +119,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ["schema", "port", "extra", "host"], diff --git a/airflow/providers/sftp/hooks/sftp.py b/airflow/providers/sftp/hooks/sftp.py index 23afa6c2d0dc5..49b063f9b26ad 100644 --- a/airflow/providers/sftp/hooks/sftp.py +++ b/airflow/providers/sftp/hooks/sftp.py @@ -19,7 +19,7 @@ import datetime import stat import warnings -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import pysftp import tenacity @@ -61,7 +61,7 @@ class SFTPHook(SSHHook): hook_name = 'SFTP' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: return { "hidden_fields": ['schema'], "relabeling": { diff --git a/airflow/providers/snowflake/hooks/snowflake.py b/airflow/providers/snowflake/hooks/snowflake.py index a61f6e0af13ce..9fa13f67a049f 100644 --- a/airflow/providers/snowflake/hooks/snowflake.py +++ b/airflow/providers/snowflake/hooks/snowflake.py @@ -97,7 +97,7 @@ def get_connection_form_widgets() -> Dict[str, Any]: } @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" import json diff --git a/airflow/providers/ssh/hooks/ssh.py b/airflow/providers/ssh/hooks/ssh.py index db98aad42f519..39b64aac787d8 100644 --- a/airflow/providers/ssh/hooks/ssh.py +++ b/airflow/providers/ssh/hooks/ssh.py @@ -88,7 +88,7 @@ class SSHHook(BaseHook): hook_name = 'SSH' @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['schema'], diff --git a/airflow/providers/yandex/hooks/yandex.py b/airflow/providers/yandex/hooks/yandex.py index 84cc3ee45641b..a337954496241 100644 --- a/airflow/providers/yandex/hooks/yandex.py +++ b/airflow/providers/yandex/hooks/yandex.py @@ -95,7 +95,7 @@ def provider_user_agent(cls) -> Optional[str]: return None @staticmethod - def get_ui_field_behaviour() -> Dict: + def get_ui_field_behaviour() -> Dict[str, Any]: """Returns custom field behaviour""" return { "hidden_fields": ['host', 'schema', 'login', 'password', 'port', 'extra'], diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py index b5b84aa21e8df..a36b592d37d23 100644 --- a/airflow/providers_manager.py +++ b/airflow/providers_manager.py @@ -18,7 +18,6 @@ """Manages all providers.""" import fnmatch import functools -import importlib import json import logging import os @@ -27,11 +26,26 @@ from collections import OrderedDict from functools import wraps from time import perf_counter -from typing import Any, Callable, Dict, List, MutableMapping, NamedTuple, Optional, Set, TypeVar, Union, cast +from typing import ( + Any, + Callable, + Dict, + List, + MutableMapping, + NamedTuple, + Optional, + Set, + Type, + TypeVar, + Union, + cast, +) import jsonschema from packaging import version as packaging_version +from airflow.exceptions import AirflowOptionalProviderFeatureException +from airflow.hooks.base import BaseHook from airflow.utils import yaml from airflow.utils.entry_points import entry_points_with_dist from airflow.utils.log.logging_mixin import LoggingMixin @@ -124,7 +138,7 @@ def _check_builtin_provider_prefix(provider_package: str, class_name: str) -> bo return True -def _sanity_check(provider_package: str, class_name: str) -> bool: +def _sanity_check(provider_package: str, class_name: str) -> Optional[Type[BaseHook]]: """ Performs coherence check on provider classes. For apache-airflow providers - it checks if it starts with appropriate package. For all providers @@ -134,31 +148,42 @@ def _sanity_check(provider_package: str, class_name: str) -> bool: :param provider_package: name of the provider package :param class_name: name of the class to import - :return True if the class is OK, False otherwise. + :return the class if the class is OK, None otherwise. """ if not _check_builtin_provider_prefix(provider_package, class_name): - return False + return None try: - import_string(class_name) + imported_class = import_string(class_name) + except AirflowOptionalProviderFeatureException as e: + # When the provider class raises AirflowOptionalProviderFeatureException + # this is an expected case when only some classes in provider are + # available. We just log debug level here + log.debug( + "Optional feature disabled on exception when importing '%s' from '%s' package", + class_name, + provider_package, + exc_info=e, + ) + return None except ImportError as e: # When there is an ImportError we turn it into debug warnings as this is # an expected case when only some providers are installed - log.debug( - "Exception when importing '%s' from '%s' package: %s", + log.warning( + "Exception when importing '%s' from '%s' package", class_name, provider_package, - e, + exc_info=e, ) - return False + return None except Exception as e: log.warning( - "Exception when importing '%s' from '%s' package: %s", + "Exception when importing '%s' from '%s' package", class_name, provider_package, - e, + exc_info=e, ) - return False - return True + return None + return imported_class class ProviderInfo(NamedTuple): @@ -631,11 +656,11 @@ def _import_hook( f"Provider package name is not set when hook_class_name ({hook_class_name}) is used" ) allowed_field_classes = [IntegerField, PasswordField, StringField, BooleanField] - if not _sanity_check(package_name, hook_class_name): + hook_class = _sanity_check(package_name, hook_class_name) + if hook_class is None: return None try: module, class_name = hook_class_name.rsplit('.', maxsplit=1) - hook_class = getattr(importlib.import_module(module), class_name) # Do not use attr here. We want to check only direct class fields not those # inherited from parent hook. This way we add form fields only once for the whole # hierarchy and we add it only from the parent hook that provides those! diff --git a/docs/apache-airflow-providers/howto/create-update-providers.rst b/docs/apache-airflow-providers/howto/create-update-providers.rst index 5ad4c65912969..f953f46195ea0 100644 --- a/docs/apache-airflow-providers/howto/create-update-providers.rst +++ b/docs/apache-airflow-providers/howto/create-update-providers.rst @@ -304,6 +304,53 @@ main Airflow documentation that involves some steps with the providers is also w ./breeze build-docs -- --package-filter apache-airflow-providers- ./breeze build-docs -- --package-filter apache-airflow +Optional provider features +-------------------------- + + .. note:: + + This feature is available in Airflow 2.3+. + +Some providers might provide optional features, which are only available when some packages or libraries +are installed. Such features will typically result in ``ImportErrors`` however those import errors +should be silently ignored rather than pollute the logs of Airflow with false warnings. False warnings +are a very bad pattern, as they tend to turn into blind spots, so avoiding false warnings is encouraged. +However until Airflow 2.3, Airflow had no mechanism to selectively ignore "known" ImportErrors. So +Airflow 2.1 and 2.2 silently ignored all ImportErrors coming from providers with actually lead to +ignoring even important import errors - without giving the clue to Airflow users that there is something +missing in provider dependencies. + +In Airflow 2.3, new exception :class:`~airflow.exceptions.OptionalProviderFeatureException` has been +introduced and Providers can use the exception to signal that the ImportError (or any other error) should +be ignored by Airflow ProvidersManager. However this Exception is only available in Airflow 2.3 so if +providers would like to remain compatible with Airflow 2.1 and 2.2, they should continue throwing +the ImportError exception. + +Example code (from Plyvel Hook, part of the Google Provider) explains how such conditional error handling +should be implemented to keep compatibility with Airflow 2.1 and 2.2 + + .. code-block:: python + + try: + import plyvel + from plyvel import DB + + from airflow.exceptions import AirflowException + from airflow.hooks.base import BaseHook + + except ImportError as e: + # Plyvel is an optional feature and if imports are missing, it should be silently ignored + # As of Airflow 2.3 and above the operator can throw OptionalProviderFeatureException + try: + from airflow.exceptions import AirflowOptionalProviderFeatureException + except ImportError: + # However, in order to keep backwards-compatibility with Airflow 2.1 and 2.2, if the + # 2.3 exception cannot be imported, the original ImportError should be raised. + # This try/except can be removed when the provider depends on Airflow >= 2.3.0 + raise e + raise AirflowOptionalProviderFeatureException(e) + + How-to Update a community provider ---------------------------------- diff --git a/tests/core/test_providers_manager.py b/tests/core/test_providers_manager.py index 6f7b709356dd4..b38ec8d3b5661 100644 --- a/tests/core/test_providers_manager.py +++ b/tests/core/test_providers_manager.py @@ -17,14 +17,13 @@ # under the License. import logging import re -import sys import unittest -from unittest.mock import call, patch +from unittest.mock import patch import pytest -import airflow -from airflow.providers_manager import ProviderInfo, ProvidersManager +from airflow.exceptions import AirflowOptionalProviderFeatureException +from airflow.providers_manager import HookClassProvider, ProviderInfo, ProvidersManager class TestProviderManager(unittest.TestCase): @@ -116,13 +115,7 @@ def test_warning_logs_not_generated(self): assert not self._caplog.records assert "sftp" in providers_manager.hooks - @patch('airflow.providers_manager.importlib.import_module') - def test_hooks(self, mock_import_module): - # importlib.resources relies on importing the containing module to read - # its content. The provider manager needs to read two validation schemas - # 'provider_info.schema.json' and 'customized_form_field_behaviours.schema.json'. - mock_import_module.side_effect = [airflow, airflow] - + def test_hooks(self): with pytest.warns(expected_warning=None) as warning_records: with self._caplog.at_level(logging.WARNING): provider_manager = ProvidersManager() @@ -131,16 +124,6 @@ def test_hooks(self, mock_import_module): assert [] == [w.message for w in warning_records.list if "hook-class-names" in str(w.message)] assert len(self._caplog.records) == 0 - # The stdlib importlib.resources implementation in 3.9 does not rely on - # importlib.import_module, so the function should not be called. The - # implementation for 3.10+ and the backport we use for up to 3.8 uses it - # to import the top-level 'airflow' for the two schema files. Also see - # comment on mocking at the beginning of the function. - if sys.version_info[:2] == (3, 9): - assert mock_import_module.mock_calls == [] - else: - assert mock_import_module.mock_calls == [call("airflow"), call("airflow")] - def test_hook_values(self): with pytest.warns(expected_warning=None) as warning_records: with self._caplog.at_level(logging.WARNING): @@ -179,3 +162,32 @@ def test_auth_backends(self): provider_manager = ProvidersManager() auth_backend_module_names = list(provider_manager.auth_backend_module_names) assert len(auth_backend_module_names) > 0 + + @patch("airflow.providers_manager.import_string") + def test_optional_feature_no_warning(self, mock_importlib_import_string): + with self._caplog.at_level(logging.WARNING): + mock_importlib_import_string.side_effect = AirflowOptionalProviderFeatureException() + providers_manager = ProvidersManager() + providers_manager._hook_provider_dict["test_connection"] = HookClassProvider( + package_name="test_package", hook_class_name="HookClass" + ) + providers_manager._import_hook( + hook_class_name=None, package_name=None, connection_type="test_connection" + ) + assert [] == self._caplog.messages + + @patch("airflow.providers_manager.import_string") + def test_optional_feature_debug(self, mock_importlib_import_string): + with self._caplog.at_level(logging.DEBUG): + mock_importlib_import_string.side_effect = AirflowOptionalProviderFeatureException() + providers_manager = ProvidersManager() + providers_manager._hook_provider_dict["test_connection"] = HookClassProvider( + package_name="test_package", hook_class_name="HookClass" + ) + providers_manager._import_hook( + hook_class_name=None, package_name=None, connection_type="test_connection" + ) + assert [ + "Optional feature disabled on exception when importing 'HookClass' from " + "'test_package' package" + ] == self._caplog.messages