Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Raise error upon DAG import instead of DAG trigger for incorrect pool_slots value in mapped tasks using partial() #39724

Merged
merged 9 commits into from
Oct 2, 2024
5 changes: 5 additions & 0 deletions airflow/decorators/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,11 @@ def _expand(self, expand_input: ExpandInput, *, strict: bool) -> XComArg:
end_date = timezone.convert_to_utc(partial_kwargs.pop("end_date", None))
if partial_kwargs.get("pool") is None:
partial_kwargs["pool"] = Pool.DEFAULT_POOL_NAME
if partial_kwargs.get("pool_slots") and partial_kwargs["pool_slots"] < 1:
RNHTTR marked this conversation as resolved.
Show resolved Hide resolved
dag_str = ""
if partial_kwargs.get("dag"):
dag_str = f" in dag {partial_kwargs['dag'].dag_id}"
uranusjr marked this conversation as resolved.
Show resolved Hide resolved
raise ValueError(f"pool slots for {task_id}{dag_str} cannot be less than 1")
partial_kwargs["retries"] = parse_retries(partial_kwargs.get("retries", DEFAULT_RETRIES))
partial_kwargs["retry_delay"] = coerce_timedelta(
partial_kwargs.get("retry_delay", DEFAULT_RETRY_DELAY),
Expand Down
5 changes: 5 additions & 0 deletions airflow/models/baseoperator.py
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,11 @@ def partial(
partial_kwargs["end_date"] = timezone.convert_to_utc(partial_kwargs["end_date"])
if partial_kwargs["pool"] is None:
partial_kwargs["pool"] = Pool.DEFAULT_POOL_NAME
if partial_kwargs["pool_slots"] < 1:
dag_str = ""
if partial_kwargs["dag"]:
dag_str = f" in dag {partial_kwargs['dag'].dag_id}"
raise ValueError(f"pool slots for {partial_kwargs['task_id']}{dag_str} cannot be less than 1")
partial_kwargs["retries"] = parse_retries(partial_kwargs["retries"])
partial_kwargs["retry_delay"] = coerce_timedelta(partial_kwargs["retry_delay"], key="retry_delay")
if partial_kwargs["max_retry_delay"] is not None:
Expand Down
9 changes: 9 additions & 0 deletions tests/models/test_mappedoperator.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,15 @@ def test_partial_on_class_invalid_ctor_args() -> None:
MockOperator.partial(task_id="a", foo="bar", bar=2)


def test_partial_on_invalid_pool_slots_raises() -> None:
"""Test that when we pass an invalid value to pool_slots in partial(),

i.e. if the value is not an integer, an error is raised at import time."""

with pytest.raises(TypeError, match="'<' not supported between instances of 'str' and 'int'"):
MockOperator.partial(task_id="pool_slots_test", pool="test", pool_slots="a").expand(arg1=[1, 2, 3])


@pytest.mark.parametrize(
["num_existing_tis", "expected"],
(
Expand Down