Bring back Pydantic 1 compatibility (#34081)

The #33956 and #33998 updated limits for Pydantic to Pydantic 2
only and removed Pydantic 1 compatibility. However it turns out
that some of important 3rd-party libraries have not yet upgraded
and it will make it impossible to install them on Airflow 2.7.1+
if we keep the limit.

For now we bring back Pydantic 1 compatibility, we remove the limit
and filter the warnings that made us remove the compatibility code.

(cherry picked from commit 30ddfc5)

airflow/configuration.py

@@ -58,6 +58,10 @@
     warnings.filterwarnings(action="default", category=DeprecationWarning, module="airflow")
     warnings.filterwarnings(action="default", category=PendingDeprecationWarning, module="airflow")
 
+    # Temporarily suppress warnings from pydantic until we upgrade minimum version of pydantic to v2
+    # Which should happen in Airflow 2.8.0
+    warnings.filterwarnings(action="ignore", category=UserWarning, module=r"pydantic._internal._config")
+
 _SQLITE3_VERSION_PATTERN = re2.compile(r"(?P<version>^\d+(?:\.\d+)*)\D?.*$")
 
 ConfigType = Union[str, int, float, bool]

airflow/serialization/pydantic/dataset.py

@@ -32,6 +32,7 @@ class Config:
         """Make sure it deals automatically with SQLAlchemy ORM classes."""
 
         from_attributes = True
+        orm_mode = True  # Pydantic 1.x compatibility.
 
 
 class TaskOutletDatasetReferencePydantic(BaseModelPydantic):
@@ -47,6 +48,7 @@ class Config:
         """Make sure it deals automatically with SQLAlchemy ORM classes."""
 
         from_attributes = True
+        orm_mode = True  # Pydantic 1.x compatibility.
 
 
 class DatasetPydantic(BaseModelPydantic):
@@ -66,6 +68,7 @@ class Config:
         """Make sure it deals automatically with SQLAlchemy ORM classes."""
 
         from_attributes = True
+        orm_mode = True  # Pydantic 1.x compatibility.
 
 
 class DatasetEventPydantic(BaseModelPydantic):
@@ -84,3 +87,4 @@ class Config:
         """Make sure it deals automatically with SQLAlchemy ORM classes."""
 
         from_attributes = True
+        orm_mode = True  # Pydantic 1.x compatibility.

airflow/serialization/pydantic/job.py

@@ -50,3 +50,4 @@ class Config:
         """Make sure it deals automatically with SQLAlchemy ORM classes."""
 
         from_attributes = True
+        orm_mode = True  # Pydantic 1.x compatibility.

airflow/serialization/pydantic/taskinstance.py

@@ -61,6 +61,7 @@ class Config:
         """Make sure it deals automatically with SQLAlchemy ORM classes."""
 
         from_attributes = True
+        orm_mode = True  # Pydantic 1.x compatibility.
 
     def xcom_pull(
         self,

airflow/serialization/serde.py

@@ -317,7 +317,14 @@ def _is_pydantic(cls: Any) -> bool:
     Checking is done by attributes as it is significantly faster than
     using isinstance.
     """
-    return hasattr(cls, "model_config") and hasattr(cls, "model_fields") and hasattr(cls, "model_fields_set")
+    return (
+        hasattr(cls, "__validators__")
+        and hasattr(cls, "__fields__")
+        and hasattr(cls, "dict")  # Pydantic v1
+        or hasattr(cls, "model_config")
+        and hasattr(cls, "model_fields")
+        and hasattr(cls, "model_fields_set")  # Pydantic v2
+    )
 
 
 def _register():

airflow/serialization/serialized_objects.py

@@ -485,7 +485,10 @@ def serialize(
         elif use_pydantic_models and _ENABLE_AIP_44:
 
             def _pydantic_model_dump(model_cls: type[BaseModel], var: Any) -> dict[str, Any]:
-                return model_cls.model_validate(var).model_dump(mode="json")
+                try:
+                    return model_cls.model_validate(var).model_dump(mode="json")  # type: ignore[attr-defined]
+                except AttributeError:  # Pydantic 1.x compatibility.
+                    return model_cls.from_orm(var).dict()  # type: ignore[attr-defined]
 
             if isinstance(var, Job):
                 return cls._encode(_pydantic_model_dump(JobPydantic, var), type_=DAT.BASE_JOB)

setup.cfg

@@ -123,7 +123,11 @@ install_requires =
     pendulum>=2.0
     pluggy>=1.0
     psutil>=4.2.0
-    pydantic>=2.3.0
+    # We should bump it to at least pydantic>=2.3.0 when we prepare Airflow 2.8.0 release
+    # we keep Pydantic < 1 for compatibility with packages that depend on Pydantic 1
+    # We should also remove then `filterwarning` for pydantic from airflow/configuration.py
+    # and # Pydantic v1 check in airflow/serialization/serde.py
+    pydantic>=1.10.0
     pygments>=2.0.1
     pyjwt>=2.0.0
     python-daemon>=3.0.0