From d40bd9841f6b157255c5c5b735b51428a44ae915 Mon Sep 17 00:00:00 2001 From: "Ernest W. Durbin III" Date: Mon, 13 Jan 2020 15:34:43 -0500 Subject: [PATCH] introduce malware queue (#7227) * introduce malware queue * correct syntax, apparently list of tuples documented doesn't work. --- Procfile | 1 + tests/unit/test_tasks.py | 7 +++++-- warehouse/tasks.py | 7 +++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Procfile b/Procfile index 2c79f62a1e98..d30253824174 100644 --- a/Procfile +++ b/Procfile @@ -2,4 +2,5 @@ release: bin/release web: bin/start-web python -m gunicorn.app.wsgiapp -c gunicorn.conf.py warehouse.wsgi:application web-uploads: bin/start-web python -m gunicorn.app.wsgiapp -c gunicorn-uploads.conf.py warehouse.wsgi:application worker: bin/start-worker celery -A warehouse worker -Q default -l info --max-tasks-per-child 32 +worker-malware: bin/start-worker celery -A warehouse worker -Q malware -l info --max-tasks-per-child 32 worker-beat: bin/start-worker celery -A warehouse beat -S redbeat.RedBeatScheduler -l info diff --git a/tests/unit/test_tasks.py b/tests/unit/test_tasks.py index 60c43d77ca43..d18a0c0a1cf7 100644 --- a/tests/unit/test_tasks.py +++ b/tests/unit/test_tasks.py @@ -468,8 +468,11 @@ def test_includeme(env, ssl, broker_url, expected_url, transport_options): "task_serializer": "json", "accept_content": ["json", "msgpack"], "task_queue_ha_policy": "all", - "task_queues": (Queue("default", routing_key="task.#"),), - "task_routes": ([]), + "task_queues": ( + Queue("default", routing_key="task.#"), + Queue("malware", routing_key="malware.#"), + ), + "task_routes": {"warehouse.malware.tasks.*": {"queue": "malware"}}, "REDBEAT_REDIS_URL": (config.registry.settings["celery.scheduler_url"]), }.items(): assert app.conf[key] == value diff --git a/warehouse/tasks.py b/warehouse/tasks.py index a93479552349..3d83b519bc7e 100644 --- a/warehouse/tasks.py +++ b/warehouse/tasks.py @@ -188,8 +188,11 @@ def includeme(config): task_default_queue="default", task_default_routing_key="task.default", task_queue_ha_policy="all", - task_queues=(Queue("default", routing_key="task.#"),), - task_routes=([]), + task_queues=( + Queue("default", routing_key="task.#"), + Queue("malware", routing_key="malware.#"), + ), + task_routes={"warehouse.malware.tasks.*": {"queue": "malware"}}, task_serializer="json", worker_disable_rate_limits=True, REDBEAT_REDIS_URL=s["celery.scheduler_url"],