Designate a queue for running the pipeline tasks

Fix syntax error and PEP8 add space hardcode worker fixup! hardcode worker fix test fix another test fix test funtino fixup! fix test funtino fix monkeypatch fixup! fix monkeypatch fixup! fix monkeypatch
appsembler · May 24, 2021 · 8ed85ee · 8ed85ee
1 parent 5a105a8
commit 8ed85ee
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 4 deletions.
diff --git a/figures/settings/lms_production.py b/figures/settings/lms_production.py
@@ -39,6 +39,12 @@ def update_celerybeat_schedule(celerybeat_schedule_settings, figures_env_tokens)
                 hour=figures_env_tokens.get('DAILY_METRICS_IMPORT_HOUR', 2),
                 minute=figures_env_tokens.get('DAILY_METRICS_IMPORT_MINUTE', 0),
                 ),
+            'options': {
+                'queue': figures_env_tokens.get(
+                    'DAILY_METRICS_IMPORT_QUEUE',
+                    'edx.lms.core.high'
+                    )
+                },
             }
 
     if figures_env_tokens.get('ENABLE_DAILY_MAU_IMPORT', False):
@@ -48,12 +54,24 @@ def update_celerybeat_schedule(celerybeat_schedule_settings, figures_env_tokens)
                 hour=figures_env_tokens.get('DAILY_MAU_IMPORT_HOUR', 0),
                 minute=figures_env_tokens.get('DAILY_MAU_IMPORT_MINUTE', 0),
                 ),
+            'options': {
+                'queue': figures_env_tokens.get(
+                    'DAILY_MAU_IMPORT_QUEUE',
+                    'edx.lms.core.high'
+                    )
+                },
             }
 
     if figures_env_tokens.get('ENABLE_FIGURES_MONTHLY_METRICS', True):
         celerybeat_schedule_settings['figures-monthly-metrics'] = {
             'task': 'figures.tasks.run_figures_monthly_metrics',
             'schedule': crontab(0, 0, day_of_month=1),
+            'options': {
+                'queue': figures_env_tokens.get(
+                    'FIGURES_MONTHLY_METRICS_QUEUE',
+                    'edx.lms.core.high'
+                    )
+                },
             }
 
 

diff --git a/figures/tasks.py b/figures/tasks.py
@@ -380,4 +380,7 @@ def run_figures_monthly_metrics():
     """
     logger.info('Starting figures.tasks.run_figures_monthly_metrics...')
     for site in get_sites():
-        populate_monthly_metrics_for_site.delay(site_id=site.id)
+        populate_monthly_metrics_for_site.apply_async(
+            site.id,
+            queue='edx.lms.core.high'
+            )
diff --git a/tests/tasks/test_monthly_tasks.py b/tests/tasks/test_monthly_tasks.py
@@ -82,10 +82,10 @@ def test_run_figures_monthly_metrics_with_faked_subtask(transactional_db, monkey
     assert expected_sites.count()
     sites_visited = []
 
-    def fake_populate_monthly_metrics_for_site(site_id):
+    def fake_populate_monthly_metrics_for_site(site_id, queue):
         sites_visited.append(site_id)
 
-    monkeypatch.setattr('figures.tasks.populate_monthly_metrics_for_site.delay',
+    monkeypatch.setattr('figures.tasks.populate_monthly_metrics_for_site.apply_async',
                         fake_populate_monthly_metrics_for_site)
 
     run_figures_monthly_metrics()

diff --git a/tests/test_settings.py b/tests/test_settings.py
@@ -118,7 +118,7 @@ def test_daily_mau_pipeline_flag_enabled(self):
         self.settings.ENV_TOKENS['FIGURES'] = { 'ENABLE_DAILY_MAU_IMPORT': True }
         plugin_settings(self.settings)
         assert self.TASK_NAME in self.settings.CELERYBEAT_SCHEDULE
-        assert set(['task', 'schedule']) == set(
+        assert set(['task', 'schedule', 'options']) == set(
             self.settings.CELERYBEAT_SCHEDULE[self.TASK_NAME].keys())
 
     def test_daily_mau_pipeline_flag_disabled(self):