From 693b7ed329589c561c5befc7f703ab20df2a0712 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 4 Sep 2023 19:24:30 +0200 Subject: [PATCH] Bring back Pydantic 1 compatibility (#34081) The #33956 and #33998 updated limits for Pydantic to Pydantic 2 only and removed Pydantic 1 compatibility. However it turns out that some of important 3rd-party libraries have not yet upgraded and it will make it impossible to install them on Airflow 2.7.1+ if we keep the limit. For now we bring back Pydantic 1 compatibility, we remove the limit and filter the warnings that made us remove the compatibility code. (cherry picked from commit 30ddfc578fa01d616c93beba6523f5e026ce5c55) --- airflow/configuration.py | 4 ++++ airflow/serialization/pydantic/dataset.py | 4 ++++ airflow/serialization/pydantic/job.py | 1 + airflow/serialization/pydantic/taskinstance.py | 1 + airflow/serialization/serde.py | 9 ++++++++- airflow/serialization/serialized_objects.py | 5 ++++- setup.cfg | 6 +++++- 7 files changed, 27 insertions(+), 3 deletions(-) diff --git a/airflow/configuration.py b/airflow/configuration.py index 41dc8d718e0d9..92b9bd0f643f4 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -58,6 +58,10 @@ warnings.filterwarnings(action="default", category=DeprecationWarning, module="airflow") warnings.filterwarnings(action="default", category=PendingDeprecationWarning, module="airflow") + # Temporarily suppress warnings from pydantic until we upgrade minimum version of pydantic to v2 + # Which should happen in Airflow 2.8.0 + warnings.filterwarnings(action="ignore", category=UserWarning, module=r"pydantic._internal._config") + _SQLITE3_VERSION_PATTERN = re2.compile(r"(?P^\d+(?:\.\d+)*)\D?.*$") ConfigType = Union[str, int, float, bool] diff --git a/airflow/serialization/pydantic/dataset.py b/airflow/serialization/pydantic/dataset.py index 07d6b3bf88a2e..096bda6ddd5d3 100644 --- a/airflow/serialization/pydantic/dataset.py +++ b/airflow/serialization/pydantic/dataset.py @@ -32,6 +32,7 @@ class Config: """Make sure it deals automatically with SQLAlchemy ORM classes.""" from_attributes = True + orm_mode = True # Pydantic 1.x compatibility. class TaskOutletDatasetReferencePydantic(BaseModelPydantic): @@ -47,6 +48,7 @@ class Config: """Make sure it deals automatically with SQLAlchemy ORM classes.""" from_attributes = True + orm_mode = True # Pydantic 1.x compatibility. class DatasetPydantic(BaseModelPydantic): @@ -66,6 +68,7 @@ class Config: """Make sure it deals automatically with SQLAlchemy ORM classes.""" from_attributes = True + orm_mode = True # Pydantic 1.x compatibility. class DatasetEventPydantic(BaseModelPydantic): @@ -84,3 +87,4 @@ class Config: """Make sure it deals automatically with SQLAlchemy ORM classes.""" from_attributes = True + orm_mode = True # Pydantic 1.x compatibility. diff --git a/airflow/serialization/pydantic/job.py b/airflow/serialization/pydantic/job.py index 5b0e961e04153..27c8ad8ca7496 100644 --- a/airflow/serialization/pydantic/job.py +++ b/airflow/serialization/pydantic/job.py @@ -50,3 +50,4 @@ class Config: """Make sure it deals automatically with SQLAlchemy ORM classes.""" from_attributes = True + orm_mode = True # Pydantic 1.x compatibility. diff --git a/airflow/serialization/pydantic/taskinstance.py b/airflow/serialization/pydantic/taskinstance.py index 0bca3505cd8a1..71d8ba576f11c 100644 --- a/airflow/serialization/pydantic/taskinstance.py +++ b/airflow/serialization/pydantic/taskinstance.py @@ -61,6 +61,7 @@ class Config: """Make sure it deals automatically with SQLAlchemy ORM classes.""" from_attributes = True + orm_mode = True # Pydantic 1.x compatibility. def xcom_pull( self, diff --git a/airflow/serialization/serde.py b/airflow/serialization/serde.py index ba60af6010187..794ef4941f756 100644 --- a/airflow/serialization/serde.py +++ b/airflow/serialization/serde.py @@ -317,7 +317,14 @@ def _is_pydantic(cls: Any) -> bool: Checking is done by attributes as it is significantly faster than using isinstance. """ - return hasattr(cls, "model_config") and hasattr(cls, "model_fields") and hasattr(cls, "model_fields_set") + return ( + hasattr(cls, "__validators__") + and hasattr(cls, "__fields__") + and hasattr(cls, "dict") # Pydantic v1 + or hasattr(cls, "model_config") + and hasattr(cls, "model_fields") + and hasattr(cls, "model_fields_set") # Pydantic v2 + ) def _register(): diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py index 8e74788a7ae7a..26dd817862296 100644 --- a/airflow/serialization/serialized_objects.py +++ b/airflow/serialization/serialized_objects.py @@ -485,7 +485,10 @@ def serialize( elif use_pydantic_models and _ENABLE_AIP_44: def _pydantic_model_dump(model_cls: type[BaseModel], var: Any) -> dict[str, Any]: - return model_cls.model_validate(var).model_dump(mode="json") + try: + return model_cls.model_validate(var).model_dump(mode="json") # type: ignore[attr-defined] + except AttributeError: # Pydantic 1.x compatibility. + return model_cls.from_orm(var).dict() # type: ignore[attr-defined] if isinstance(var, Job): return cls._encode(_pydantic_model_dump(JobPydantic, var), type_=DAT.BASE_JOB) diff --git a/setup.cfg b/setup.cfg index fa963c297507e..367924d82ddce 100644 --- a/setup.cfg +++ b/setup.cfg @@ -123,7 +123,11 @@ install_requires = pendulum>=2.0 pluggy>=1.0 psutil>=4.2.0 - pydantic>=2.3.0 + # We should bump it to at least pydantic>=2.3.0 when we prepare Airflow 2.8.0 release + # we keep Pydantic < 1 for compatibility with packages that depend on Pydantic 1 + # We should also remove then `filterwarning` for pydantic from airflow/configuration.py + # and # Pydantic v1 check in airflow/serialization/serde.py + pydantic>=1.10.0 pygments>=2.0.1 pyjwt>=2.0.0 python-daemon>=3.0.0