diff --git a/docs/howto/blueprints.rst b/docs/howto/blueprints.rst index 3968c1006..0cbc1f0e8 100644 --- a/docs/howto/blueprints.rst +++ b/docs/howto/blueprints.rst @@ -5,16 +5,15 @@ Procrastinate provides Blueprints as a way to factor a large number of tasks into smaller self contained collections. You may want to create a collection for simple organsational reasons within the -same project. Or you may want to maintain a collection of tasks in a seperate -package which is maintained independently of you Procrastinate server and -workers. eg:: +same project. Or you may want to maintain a collection of tasks in a seperate +package which is maintained independently of your Procrastinate codebase:: ... from my_external_package import tasks_blueprint ... - app.register_blueprint(tasks_blueprint) + app.add_tasks_from(tasks_blueprint, namespace="my_external_package") Blueprints are easy to use, and task creation follows the pattern and API as @@ -36,4 +35,4 @@ In your projcet register the blueprint with the `App` after you have created it: app = App(connector=AiopgConnector()) - app.register_blueprint(my_blueprint) + app.add_tasks_from(my_blueprint, namespace="unique_name") diff --git a/tests/acceptance/app.py b/tests/acceptance/app.py index d0f8262f2..8a2e0121e 100644 --- a/tests/acceptance/app.py +++ b/tests/acceptance/app.py @@ -47,7 +47,8 @@ def sum_task(a, b): print(a + b) -app.register_blueprint(bp) +app.add_tasks_from(bp, namespace="ns") +app.add_task_alias(sum_task, "tests.acceptance.app.sum_task") @app.task(queue="default") diff --git a/tests/acceptance/test_nominal.py b/tests/acceptance/test_nominal.py index d73780902..bd9dc5e2f 100644 --- a/tests/acceptance/test_nominal.py +++ b/tests/acceptance/test_nominal.py @@ -45,6 +45,7 @@ def test_nominal(defer, worker): defer("increment_task", a=3) stdout, stderr = worker() + print(stdout, stderr) assert stdout.splitlines() == ["Launching a worker on all queues", "12", "7", "4"] assert stderr.startswith("DEBUG:procrastinate.") @@ -52,18 +53,22 @@ def test_nominal(defer, worker): defer("product_task", a=5, b=4) stdout, stderr = worker("default") + print(stdout, stderr) assert "20" not in stdout stdout, stderr = worker("product_queue") + print(stdout, stderr) assert stdout.splitlines() == ["Launching a worker on product_queue", "20"] defer("two_fails") stdout, stderr = worker() + print(stdout, stderr) assert "Print something to stdout" in stdout assert stderr.count("Exception: This should fail") == 2 defer("multiple_exception_failures") stdout, stderr = worker() + print(stdout, stderr) assert ( stdout == """Launching a worker on all queues diff --git a/tests/acceptance/test_shell.py b/tests/acceptance/test_shell.py index 2a3126695..96ededac4 100644 --- a/tests/acceptance/test_shell.py +++ b/tests/acceptance/test_shell.py @@ -16,48 +16,56 @@ def shell(process_env): yield proc -def test_shell(shell, defer): +@pytest.fixture +def write(shell): + def _(s: str): + print(s, file=shell.stdin) + + return _ + + +def test_shell(shell, write, defer): defer("sum_task", ["--lock=a"], a=5, b=7) defer("sum_task", ["--lock=lock"], a=3, b=8) defer("sum_task", ["--queue=other", "--lock=lock"], a=1, b=2) defer("increment_task", ["--lock=b"], a=5) - print("cancel 2", file=shell.stdin) - print("cancel 3", file=shell.stdin) - print("cancel 4", file=shell.stdin) + write("cancel 2") + write("cancel 3") + write("cancel 4") - print("list_jobs", file=shell.stdin) - print("list_jobs queue=other details", file=shell.stdin) - print("list_queues", file=shell.stdin) - print("list_tasks", file=shell.stdin) - print("list_locks", file=shell.stdin) + write("list_jobs") + write("list_jobs queue=other details") + write("list_queues") + write("list_tasks") + write("list_locks") - print("exit", file=shell.stdin) + write("exit") shell.stdin.close() assert shell.stdout.readlines() == [ "Welcome to the procrastinate shell. Type help or ? to list commands.\n", "\n", # cancel - "procrastinate> #2 tests.acceptance.app.sum_task on default - [failed]\n", - "procrastinate> #3 tests.acceptance.app.sum_task on other - [failed]\n", + "procrastinate> #2 ns:tests.acceptance.app.sum_task on default - [failed]\n", + "procrastinate> #3 ns:tests.acceptance.app.sum_task on other - [failed]\n", "procrastinate> #4 tests.acceptance.app.increment_task on default - [failed]\n", # list_jobs - "procrastinate> #1 tests.acceptance.app.sum_task on default - [todo]\n", - "#2 tests.acceptance.app.sum_task on default - [failed]\n", - "#3 tests.acceptance.app.sum_task on other - [failed]\n", + "procrastinate> #1 ns:tests.acceptance.app.sum_task on default - [todo]\n", + "#2 ns:tests.acceptance.app.sum_task on default - [failed]\n", + "#3 ns:tests.acceptance.app.sum_task on other - [failed]\n", "#4 tests.acceptance.app.increment_task on default - [failed]\n", # list_jobs queue=other details - "procrastinate> #3 tests.acceptance.app.sum_task on other - [failed] " + "procrastinate> #3 ns:tests.acceptance.app.sum_task on other - [failed] " "(attempts=0, scheduled_at=None, args={'a': 1, 'b': 2}, lock=lock)\n", # list_queues "procrastinate> default: 3 jobs (todo: 1, doing: 0, succeeded: 0, failed: 2)\n", "other: 1 jobs (todo: 0, doing: 0, succeeded: 0, failed: 1)\n", # list_tasks + "ns:tests.acceptance.app.sum_task: 3 jobs " + "(todo: 1, doing: 0, succeeded: 0, failed: 2)\n", "procrastinate> tests.acceptance.app.increment_task: 1 jobs " "(todo: 0, doing: 0, succeeded: 0, failed: 1)\n", - "tests.acceptance.app.sum_task: 3 jobs " - "(todo: 1, doing: 0, succeeded: 0, failed: 2)\n", "procrastinate> a: 1 jobs (todo: 1, doing: " "0, succeeded: 0, failed: 0)\n", "b: 1 jobs (todo: 0, doing: 0, succeeded: " "0, failed: 1)\n", "lock: 2 jobs (todo: 0, doing: 0, succeeded: 0, failed: 2)\n", diff --git a/tests/conftest.py b/tests/conftest.py index 801d349f2..af4e7a13a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,7 +16,7 @@ from procrastinate import aiopg_connector as aiopg_connector_module from procrastinate import app as app_module -from procrastinate import jobs +from procrastinate import builtin_tasks, jobs from procrastinate import psycopg2_connector as psycopg2_connector_module from procrastinate import schema, testing from procrastinate.contrib.sqlalchemy import ( @@ -168,7 +168,15 @@ def connector(): @pytest.fixture -def not_opened_app(connector): +def reset_builtin_task_names(): + builtin_tasks.remove_old_jobs.name = "procrastinate.builtin_tasks.remove_old_jobs" + builtin_tasks.builtin.tasks = { + task.name: task for task in builtin_tasks.builtin.tasks.values() + } + + +@pytest.fixture +def not_opened_app(connector, reset_builtin_task_names): return app_module.App(connector=connector) diff --git a/tests/integration/test_worker.py b/tests/integration/test_worker.py index ecdd39ae5..bd9ebed41 100644 --- a/tests/integration/test_worker.py +++ b/tests/integration/test_worker.py @@ -45,7 +45,6 @@ def t(): # remove the periodic_deferrer_no_task log record because that makes the test flaky logs.pop("periodic_deferrer_no_task", None) assert list(logs.items()) == [ - ("register_queue", "DEBUG"), ("about_to_defer_job", "DEBUG"), ("job_defer", "INFO"), ("loaded_job_info", "DEBUG"), diff --git a/tests/unit/test_app.py b/tests/unit/test_app.py index fe0def270..04487bbde 100644 --- a/tests/unit/test_app.py +++ b/tests/unit/test_app.py @@ -82,28 +82,15 @@ def wrapped(): def test_app_register_builtins(app): - assert app.queues == {"builtin"} assert "procrastinate.builtin_tasks.remove_old_jobs" in app.tasks - assert "remove_old_jobs" in app.builtin_tasks + assert "builtin:procrastinate.builtin_tasks.remove_old_jobs" in app.tasks def test_app_register(app): - task = tasks.Task(task_func, app=app, queue="queue", name="bla") + task = tasks.Task(task_func, blueprint=app, queue="queue", name="bla") - app._register(task) + app._register_task(task) - assert app.queues == {"queue", "builtin"} - assert "bla" in app.tasks - assert app.tasks["bla"] == task - - -def test_app_register_queue_already_exists(app): - app.queues.add("queue") - task = tasks.Task(task_func, app=app, queue="queue", name="bla") - - app._register(task) - - assert app.queues == {"queue", "builtin"} assert "bla" in app.tasks assert app.tasks["bla"] == task diff --git a/tests/unit/test_blueprints.py b/tests/unit/test_blueprints.py index fb87e0094..cbc0775d6 100644 --- a/tests/unit/test_blueprints.py +++ b/tests/unit/test_blueprints.py @@ -17,17 +17,17 @@ def test_app_task_explicit(app, mocker): def wrapped(): return "foo" - app.register_blueprint(bp) + app.add_tasks_from(bp, namespace="ns") assert wrapped() == "foo" - assert app.tasks["foobar"].name == "foobar" - assert app.tasks["foobar"].queue == "bar" - assert app.tasks["foobar"].lock == "sher" - assert app.tasks["foobar"].queueing_lock == "baz" - assert isinstance(app.tasks["foobar"].retry_strategy, retry.RetryStrategy) - assert app.tasks["foobar"].pass_context is True - assert app.tasks["foobar"] is wrapped - assert app.tasks["foobar"].func is wrapped.__wrapped__ + assert app.tasks["ns:foobar"].name == "ns:foobar" + assert app.tasks["ns:foobar"].queue == "bar" + assert app.tasks["ns:foobar"].lock == "sher" + assert app.tasks["ns:foobar"].queueing_lock == "baz" + assert isinstance(app.tasks["ns:foobar"].retry_strategy, retry.RetryStrategy) + assert app.tasks["ns:foobar"].pass_context is True + assert app.tasks["ns:foobar"] is wrapped + assert app.tasks["ns:foobar"].func is wrapped.__wrapped__ def test_app_task_implicit(app): @@ -37,12 +37,12 @@ def test_app_task_implicit(app): def wrapped(): return "foo" - app.register_blueprint(bp) + app.add_tasks_from(bp, namespace="ns") - registered_task = app.tasks["tests.unit.test_blueprints.wrapped"] + registered_task = app.tasks["ns:tests.unit.test_blueprints.wrapped"] assert "foo" == wrapped() - assert "tests.unit.test_blueprints.wrapped" == registered_task.name + assert "ns:tests.unit.test_blueprints.wrapped" == registered_task.name assert "default" == registered_task.queue assert registered_task is wrapped assert registered_task.func is wrapped.__wrapped__ diff --git a/tests/unit/test_task_creation_signatures.py b/tests/unit/test_task_creation_signatures.py deleted file mode 100644 index f2b2d6bb7..000000000 --- a/tests/unit/test_task_creation_signatures.py +++ /dev/null @@ -1,20 +0,0 @@ -import inspect - -from procrastinate import App, Blueprint -from procrastinate.protocols import TaskCreator - - -# def test_task_signatures(app, mocker): -def test_task_signatures(): - """Tasks can be created in two ways, both of which need to maintain an - identical API. This test simple test that App.task and Blueprint.task have - the same function signature. - - This is further enforced with protocols.TaskCreator and mypy checks. - """ - - # Check that both App.task and Blueprint.task implement Taskcreator.task - assert inspect.signature(App.task) == inspect.signature(TaskCreator.task) - assert inspect.signature(Blueprint.task) == inspect.signature(TaskCreator.task) - # Sanity check - assert inspect.signature(App.task) == inspect.signature(Blueprint.task) diff --git a/tests/unit/test_tasks.py b/tests/unit/test_tasks.py index e10ff816d..0a7f10047 100644 --- a/tests/unit/test_tasks.py +++ b/tests/unit/test_tasks.py @@ -10,7 +10,7 @@ def task_func(): def test_task_init_with_no_name(app): - task = tasks.Task(task_func, app=app, queue="queue") + task = tasks.Task(task_func, blueprint=app, queue="queue") assert task.func is task_func assert task.name == "tests.unit.test_tasks.task_func" @@ -18,7 +18,7 @@ def test_task_init_with_no_name(app): @pytest.mark.asyncio async def test_task_defer_async(app, connector): - task = tasks.Task(task_func, app=app, queue="queue") + task = tasks.Task(task_func, blueprint=app, queue="queue") await task.defer_async(c=3) @@ -79,7 +79,7 @@ def test_configure_task_schedule_in_and_schedule_at(job_manager): def test_task_configure(app): - task = tasks.Task(task_func, app=app, queue="queue") + task = tasks.Task(task_func, blueprint=app, queue="queue") job = task.configure(lock="sher", task_kwargs={"yay": "ho"}).job @@ -90,7 +90,7 @@ def test_task_configure(app): def test_task_configure_override_queue(app): - task = tasks.Task(task_func, app=app, queue="queue") + task = tasks.Task(task_func, blueprint=app, queue="queue") job = task.configure(queue="other_queue").job @@ -98,7 +98,7 @@ def test_task_configure_override_queue(app): def test_task_get_retry_exception_none(app): - task = tasks.Task(task_func, app=app, queue="queue") + task = tasks.Task(task_func, blueprint=app, queue="queue") job = task.configure().job assert task.get_retry_exception(exception=None, job=job) is None @@ -107,7 +107,7 @@ def test_task_get_retry_exception_none(app): def test_task_get_retry_exception(app, mocker): mock = mocker.patch("procrastinate.retry.RetryStrategy.get_retry_exception") - task = tasks.Task(task_func, app=app, queue="queue", retry=10) + task = tasks.Task(task_func, blueprint=app, queue="queue", retry=10) job = task.configure().job exception = ValueError() diff --git a/tests/unit/test_worker.py b/tests/unit/test_worker.py index 4b13c41af..bf3520ea7 100644 --- a/tests/unit/test_worker.py +++ b/tests/unit/test_worker.py @@ -193,7 +193,7 @@ def task_func(a, b): # pylint: disable=unused-argument result.append(s) return s - task = tasks.Task(task_func, app=app, queue="yay", name="job") + task = tasks.Task(task_func, blueprint=app, queue="yay", name="job") app.tasks = {"task_func": task} @@ -251,7 +251,7 @@ async def test_run_job_error(app): def job(a, b): # pylint: disable=unused-argument raise ValueError("nope") - task = tasks.Task(job, app=app, queue="yay", name="job") + task = tasks.Task(job, blueprint=app, queue="yay", name="job") task.func = job app.tasks = {"job": task} @@ -273,7 +273,7 @@ async def test_run_job_retry(app): def job(a, b): # pylint: disable=unused-argument raise ValueError("nope") - task = tasks.Task(job, app=app, queue="yay", name="job", retry=True) + task = tasks.Task(job, blueprint=app, queue="yay", name="job", retry=True) task.func = job app.tasks = {"job": task}