Skip to content

Commit

Permalink
Enables alternate Celery daily task for Figures
Browse files Browse the repository at this point in the history
This commit adds ENV_TOKENS['FIGURES']['DAILY_TASK'] override to define
which Celery task is called by CeleryBeat to run the daily data update
  • Loading branch information
johnbaldwin committed Feb 17, 2022
1 parent 4315bf0 commit 926cd7c
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 4 deletions.
6 changes: 5 additions & 1 deletion figures/settings/lms_production.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import os
from celery.schedules import crontab

FIGURES_DEFAULT_DAILY_TASK = 'figures.tasks.populate_daily_metrics'


class FiguresRouter(object):

Expand Down Expand Up @@ -64,8 +66,10 @@ def update_celerybeat_schedule(
https://stackoverflow.com/questions/51631455/how-to-route-tasks-to-different-queues-with-celery-and-django
"""
if figures_env_tokens.get('ENABLE_DAILY_METRICS_IMPORT', True):
figures_daily_task = figures_env_tokens.get('DAILY_TASK',
FIGURES_DEFAULT_DAILY_TASK)
celerybeat_schedule_settings['figures-populate-daily-metrics'] = {
'task': 'figures.tasks.populate_daily_metrics',
'task': figures_daily_task,
'schedule': crontab(
hour=figures_env_tokens.get('DAILY_METRICS_IMPORT_HOUR', 2),
minute=figures_env_tokens.get('DAILY_METRICS_IMPORT_MINUTE', 0),
Expand Down
47 changes: 44 additions & 3 deletions tests/test_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@


from figures import helpers as figures_helpers
from figures.settings.lms_production import plugin_settings
from figures.settings.lms_production import (
FIGURES_DEFAULT_DAILY_TASK,
plugin_settings
)


@pytest.mark.parametrize('features, expected', [
Expand Down Expand Up @@ -90,6 +93,44 @@ def test_update_settings(self, figures_env_tokens, run_celery):
assert settings.ENV_TOKENS['FIGURES'] == figures_env_tokens


class TestDailyTaskFunction(object):
"""Tests setting for Figures top level Celery daily job task
The top level Celery daily job task is what the scheduler calls to populate
Figures enrollment data snapshots and Figures daily aggregate metrics
We added this test case so that we can improve the daily Figures jobs, while
retaining the ability for deployments to run the previously implemented
daily jobs by default. This is a risk reduction strategy
"""
ALTERNATE_TASK = 'figures.tasks.populate_daily_metrics_v2'

@pytest.fixture(autouse=True)
def setup(self, db):
self.settings = mock.Mock(
WEBPACK_LOADER={},
CELERYBEAT_SCHEDULE={},
FEATURES={},
ENV_TOKENS={},
CELERY_IMPORTS=[],
)

def test_uses_default_daily_task(self):
"""Do the default settings define the default Celery task in CeleryBeat?
"""
plugin_settings(self.settings)
task_found = self.settings.CELERYBEAT_SCHEDULE['figures-populate-daily-metrics']['task']
assert task_found == FIGURES_DEFAULT_DAILY_TASK

def test_uses_env_extra_daily_task(self):
"""Does overriding the Celery task function setting in CeleryBeat?
"""
self.settings.ENV_TOKENS['FIGURES'] = {'DAILY_TASK': self.ALTERNATE_TASK}
plugin_settings(self.settings)
task_found = self.settings.CELERYBEAT_SCHEDULE['figures-populate-daily-metrics']['task']
assert task_found == self.ALTERNATE_TASK


class TestDailyMauPipelineSettings(object):
"""Tests MAU pipeline settings
Expand All @@ -115,14 +156,14 @@ def setup(self, db):
)

def test_daily_mau_pipeline_flag_enabled(self):
self.settings.ENV_TOKENS['FIGURES'] = { 'ENABLE_DAILY_MAU_IMPORT': True }
self.settings.ENV_TOKENS['FIGURES'] = {'ENABLE_DAILY_MAU_IMPORT': True}
plugin_settings(self.settings)
assert self.TASK_NAME in self.settings.CELERYBEAT_SCHEDULE
assert set(['task', 'schedule', 'options']) == set(
self.settings.CELERYBEAT_SCHEDULE[self.TASK_NAME].keys())

def test_daily_mau_pipeline_flag_disabled(self):
self.settings.ENV_TOKENS['FIGURES'] = { 'ENABLE_DAILY_MAU_IMPORT': False }
self.settings.ENV_TOKENS['FIGURES'] = {'ENABLE_DAILY_MAU_IMPORT': False}
plugin_settings(self.settings)
assert self.TASK_NAME not in self.settings.CELERYBEAT_SCHEDULE

Expand Down

0 comments on commit 926cd7c

Please sign in to comment.