From 8ed85ee7742ff9ad6f8b951c2c3b020b07b362c4 Mon Sep 17 00:00:00 2001 From: melvinsoft Date: Fri, 21 May 2021 12:51:55 -0300 Subject: [PATCH] Designate a queue for running the pipeline tasks Fix syntax error and PEP8 add space hardcode worker fixup! hardcode worker fix test fix another test fix test funtino fixup! fix test funtino fix monkeypatch fixup! fix monkeypatch fixup! fix monkeypatch --- figures/settings/lms_production.py | 18 ++++++++++++++++++ figures/tasks.py | 5 ++++- tests/tasks/test_monthly_tasks.py | 4 ++-- tests/test_settings.py | 2 +- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/figures/settings/lms_production.py b/figures/settings/lms_production.py index 5f9217f5..14fd5333 100644 --- a/figures/settings/lms_production.py +++ b/figures/settings/lms_production.py @@ -39,6 +39,12 @@ def update_celerybeat_schedule(celerybeat_schedule_settings, figures_env_tokens) hour=figures_env_tokens.get('DAILY_METRICS_IMPORT_HOUR', 2), minute=figures_env_tokens.get('DAILY_METRICS_IMPORT_MINUTE', 0), ), + 'options': { + 'queue': figures_env_tokens.get( + 'DAILY_METRICS_IMPORT_QUEUE', + 'edx.lms.core.high' + ) + }, } if figures_env_tokens.get('ENABLE_DAILY_MAU_IMPORT', False): @@ -48,12 +54,24 @@ def update_celerybeat_schedule(celerybeat_schedule_settings, figures_env_tokens) hour=figures_env_tokens.get('DAILY_MAU_IMPORT_HOUR', 0), minute=figures_env_tokens.get('DAILY_MAU_IMPORT_MINUTE', 0), ), + 'options': { + 'queue': figures_env_tokens.get( + 'DAILY_MAU_IMPORT_QUEUE', + 'edx.lms.core.high' + ) + }, } if figures_env_tokens.get('ENABLE_FIGURES_MONTHLY_METRICS', True): celerybeat_schedule_settings['figures-monthly-metrics'] = { 'task': 'figures.tasks.run_figures_monthly_metrics', 'schedule': crontab(0, 0, day_of_month=1), + 'options': { + 'queue': figures_env_tokens.get( + 'FIGURES_MONTHLY_METRICS_QUEUE', + 'edx.lms.core.high' + ) + }, } diff --git a/figures/tasks.py b/figures/tasks.py index bba37868..e85717a5 100644 --- a/figures/tasks.py +++ b/figures/tasks.py @@ -380,4 +380,7 @@ def run_figures_monthly_metrics(): """ logger.info('Starting figures.tasks.run_figures_monthly_metrics...') for site in get_sites(): - populate_monthly_metrics_for_site.delay(site_id=site.id) + populate_monthly_metrics_for_site.apply_async( + site.id, + queue='edx.lms.core.high' + ) diff --git a/tests/tasks/test_monthly_tasks.py b/tests/tasks/test_monthly_tasks.py index 5f39877f..2e1546cb 100644 --- a/tests/tasks/test_monthly_tasks.py +++ b/tests/tasks/test_monthly_tasks.py @@ -82,10 +82,10 @@ def test_run_figures_monthly_metrics_with_faked_subtask(transactional_db, monkey assert expected_sites.count() sites_visited = [] - def fake_populate_monthly_metrics_for_site(site_id): + def fake_populate_monthly_metrics_for_site(site_id, queue): sites_visited.append(site_id) - monkeypatch.setattr('figures.tasks.populate_monthly_metrics_for_site.delay', + monkeypatch.setattr('figures.tasks.populate_monthly_metrics_for_site.apply_async', fake_populate_monthly_metrics_for_site) run_figures_monthly_metrics() diff --git a/tests/test_settings.py b/tests/test_settings.py index 4771727a..e9192377 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -118,7 +118,7 @@ def test_daily_mau_pipeline_flag_enabled(self): self.settings.ENV_TOKENS['FIGURES'] = { 'ENABLE_DAILY_MAU_IMPORT': True } plugin_settings(self.settings) assert self.TASK_NAME in self.settings.CELERYBEAT_SCHEDULE - assert set(['task', 'schedule']) == set( + assert set(['task', 'schedule', 'options']) == set( self.settings.CELERYBEAT_SCHEDULE[self.TASK_NAME].keys()) def test_daily_mau_pipeline_flag_disabled(self):