Pipeline improvement prerequisites

devsite/.env.example

@@ -15,8 +15,8 @@ DATABASE_URL=mysql://figures_user:[email protected]:3306/figures-db
 # Set which expected Open edX release mocks for devsite to use.
 # Valid options are: "GINKGO", "HAWTHORN", "JUNIPER"
 #
-# If not specified here, then "HAWTHORN" is used
-OPENEDX_RELEASE=JUNIPER
+# If not specified here, then "JUNIPER" is used
+# OPENEDX_RELEASE=JUNIPER
 
 # Enable/disable Figures multisite mode in devsite
 # This also requires

devsite/devsite/settings.py

@@ -14,6 +14,7 @@
 from figures.settings.lms_production import (
     update_webpack_loader,
     update_celerybeat_schedule,
+    update_celery_routes,
 )
 
 DEVSITE_BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -22,7 +23,7 @@
 env = environ.Env(
     DEBUG=(bool, True),
     ALLOWED_HOSTS=(list, []),
-    OPENEDX_RELEASE=(str, 'HAWTHORN'),
+    OPENEDX_RELEASE=(str, 'JUNIPER'),
     FIGURES_IS_MULTISITE=(bool, False),
     ENABLE_DEVSITE_CELERY=(bool, True),
     ENABLE_OPENAPI_DOCS=(bool, False),
@@ -240,8 +241,16 @@
 # We have an empty dict here to replicate behavior in the LMS
 ENV_TOKENS = {}
 
+PRJ_SETTINGS = {
+    'CELERY_ROUTES': "app.celery.routes"
+}
+
+FIGURES_PIPELINE_TASKS_ROUTING_KEY = ""
+
 update_webpack_loader(WEBPACK_LOADER, ENV_TOKENS)
-update_celerybeat_schedule(CELERYBEAT_SCHEDULE, ENV_TOKENS)
+update_celerybeat_schedule(CELERYBEAT_SCHEDULE, ENV_TOKENS, FIGURES_PIPELINE_TASKS_ROUTING_KEY)
+update_celery_routes(PRJ_SETTINGS, ENV_TOKENS, FIGURES_PIPELINE_TASKS_ROUTING_KEY)
+
 
 # Used by Django Debug Toolbar
 INTERNAL_IPS = [

devsite/devsite/test_settings.py

@@ -28,7 +28,7 @@ def root(*args):
 
 
 env = environ.Env(
-    OPENEDX_RELEASE=(str, 'HAWTHORN'),
+    OPENEDX_RELEASE=(str, 'JUNIPER'),
 )
 
 environ.Env.read_env(join(dirname(dirname(__file__)), '.env'))

figures/helpers.py

@@ -1,6 +1,7 @@
 """Helper functions to make data handling and conversions easier
 
 # Figures 0.3.13 - Defining scope of this module
+# Figures 0.4.x - Yep, this module is still handy and the scope hasn't exploded
 
 The purpose of this module is to provide conveniece methods around commonly
 executed statements. These convenience methods should serve as shorthand for
@@ -34,6 +35,8 @@
 
 ## What does not belong here?
 
+* Most importantly, if you have to import from another figures module, it does
+  not belong here!
 * "Feature" functionality does not belong here
 * Long functions do not belong here
 * Code that communicates outside of Figures does not belong here. No database,
@@ -42,7 +45,10 @@
 This is not an exhaustive list. We'll grow it as needed.
 
 An important point is that we should not expect this module to be a permanent
-home for functionality.
+home for the functionality included. As we evolve Figures, we may find functions
+here that have a stronger context with another module. For example, we've got
+a decent set of date oriented functions that are candidatdes for a datetime and
+date handling module.
 """
 
 from __future__ import absolute_import
@@ -188,6 +194,15 @@ def prev_day(val):
     return days_from(val, -1)
 
 
+def utc_yesterday():
+    """Get "yesterday" form the utc datetime
+
+    We primarily use this for the daily metrics collection. However, it proves
+    handy as a convenience function for exploring data in the Django shell
+    """
+    return prev_day(datetime.datetime.utcnow().date())
+
+
 def days_in_month(month_for):
     _, num_days_in_month = calendar.monthrange(month_for.year, month_for.month)
     return num_days_in_month

figures/migrations/0016_add_collect_elapsed_to_ed_and_lcgm.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('figures', '0015_add_enrollment_data_model'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='enrollmentdata',
+            name='collect_elapsed',
+            field=models.FloatField(null=True),
+        ),
+        migrations.AddField(
+            model_name='learnercoursegrademetrics',
+            name='collect_elapsed',
+            field=models.FloatField(null=True),
+        ),
+    ]

figures/models.py

@@ -5,6 +5,7 @@
 
 from __future__ import absolute_import
 from datetime import date
+import timeit
 from django.conf import settings
 from django.contrib.sites.models import Site
 from django.core.validators import MaxValueValidator, MinValueValidator
@@ -17,7 +18,7 @@
 from model_utils.models import TimeStampedModel
 
 from figures.compat import CourseEnrollment
-from figures.helpers import as_course_key
+from figures.helpers import as_course_key, utc_yesterday
 from figures.progress import EnrollmentProgress
 
 
@@ -203,7 +204,7 @@ class EnrollmentDataManager(models.Manager):
     EnrollmentData instances.
 
     """
-    def set_enrollment_data(self, site, user, course_id, course_enrollment=False):
+    def set_enrollment_data(self, site, user, course_id, course_enrollment=None):
         """
         This is an expensive call as it needs to call CourseGradeFactory if
         there is not already a LearnerCourseGradeMetrics record for the learner
@@ -259,6 +260,71 @@ def set_enrollment_data(self, site, user, course_id, course_enrollment=False):
             defaults=defaults)
         return obj, created
 
+    def update_metrics(self, site, course_enrollment, force_update=False):
+        """
+        This is an expensive call as it needs to call CourseGradeFactory if
+        there is not already a LearnerCourseGradeMetrics record for the learner
+
+        """
+        date_for = utc_yesterday()
+
+        # check if we already have a record for the date for EnrollmentData
+
+        # Alternately, we could use a try/except on a 'get' call, however, this
+        # would be much slower for a bunch of new enrollments
+
+        # Should only be one even if we don't include the site in the query
+        # because course_id should be globally unique
+        # If course id is ever NOT globally unique, then we need to add site
+        # to the query
+        ed_recs = EnrollmentData.objects.filter(
+            user_id=course_enrollment.user_id,
+            course_id=str(course_enrollment.course_id))
+
+        if not ed_recs or ed_recs[0].date_for < date_for or force_update:
+            # We do the update
+            start_time = timeit.default_timer()
+            # get the progress data
+            ep = EnrollmentProgress(user=course_enrollment.user,
+                                    course_id=str(course_enrollment.course_id))
+            defaults = dict(
+                date_for=date_for,
+                is_completed=ep.is_completed(),
+                progress_percent=ep.progress_percent(),
+                points_possible=ep.progress.get('points_possible', 0),
+                points_earned=ep.progress.get('points_earned', 0),
+                sections_possible=ep.progress.get('sections_possible', 0),
+                sections_worked=ep.progress.get('sections_worked', 0),
+                is_enrolled=course_enrollment.is_active,
+                date_enrolled=course_enrollment.created,
+            )
+            elapsed = timeit.default_timer() - start_time
+            defaults['collect_elapsed'] = elapsed
+
+            ed_rec, created = self.update_or_create(
+                site=site,
+                user=course_enrollment.user,
+                course_id=str(course_enrollment.course_id),
+                defaults=defaults)
+            # create a new LCGM record
+            # if it already exists for the day
+            LearnerCourseGradeMetrics.objects.update_or_create(
+                site=ed_rec.site,
+                user=ed_rec.user,
+                course_id=ed_rec.course_id,
+                date_for=date_for,
+                defaults=dict(
+                    points_possible=ed_rec.points_possible,
+                    points_earned=ed_rec.points_earned,
+                    sections_worked=ed_rec.sections_worked,
+                    sections_possible=ed_rec.sections_possible,
+                    collect_elapsed=elapsed
+                )
+            )
+            return ed_rec, created
+        else:
+            return ed_recs[0], False
+
 
 @python_2_unicode_compatible
 class EnrollmentData(TimeStampedModel):
@@ -301,6 +367,9 @@ class EnrollmentData(TimeStampedModel):
     sections_worked = models.IntegerField()
     sections_possible = models.IntegerField()
 
+    # seconds it took to collect progress data
+    collect_elapsed = models.FloatField(null=True)
+
     objects = EnrollmentDataManager()
 
     class Meta:
@@ -324,7 +393,7 @@ def progress_details(self):
 
 
 class LearnerCourseGradeMetricsManager(models.Manager):
-    """Custom model manager for LearnerCourseGrades model
+    """Custom model manager for LearnerCourseGradeMetrics model
     """
     def latest_lcgm(self, user, course_id):
         """Gets the most recent record for the given user and course
@@ -414,9 +483,9 @@ class LearnerCourseGradeMetrics(TimeStampedModel):
     Purpose is primarliy to improve performance for the front end. In addition,
     data collected can be used for course progress over time
 
-    We're capturing data from figures.metrics.LearnerCourseGrades
+    We're capturing data from figures.progress.EnrollmentProgress
 
-    Note: We're probably going to move ``LearnerCourseGrades`` to figures.pipeline
+    Note: We're probably going to move ``EnrollmentProgress`` to figures.pipeline
     since that class will only be needed by the pipeline
 
     Even though this is for a course enrollment, we're mapping to the user
@@ -452,6 +521,9 @@ class LearnerCourseGradeMetrics(TimeStampedModel):
     sections_worked = models.IntegerField()
     sections_possible = models.IntegerField()
 
+    # seconds it took to collect progress data
+    collect_elapsed = models.FloatField(null=True)
+
     objects = LearnerCourseGradeMetricsManager()
 
     class Meta:

figures/settings/lms_production.py

@@ -5,6 +5,8 @@
 import os
 from celery.schedules import crontab
 
+FIGURES_DEFAULT_DAILY_TASK = 'figures.tasks.populate_daily_metrics'
+
 
 class FiguresRouter(object):
 
@@ -64,8 +66,10 @@ def update_celerybeat_schedule(
     https://stackoverflow.com/questions/51631455/how-to-route-tasks-to-different-queues-with-celery-and-django
     """
     if figures_env_tokens.get('ENABLE_DAILY_METRICS_IMPORT', True):
+        figures_daily_task = figures_env_tokens.get('DAILY_TASK',
+                                                    FIGURES_DEFAULT_DAILY_TASK)
         celerybeat_schedule_settings['figures-populate-daily-metrics'] = {
-            'task': 'figures.tasks.populate_daily_metrics',
+            'task': figures_daily_task,
             'schedule': crontab(
                 hour=figures_env_tokens.get('DAILY_METRICS_IMPORT_HOUR', 2),
                 minute=figures_env_tokens.get('DAILY_METRICS_IMPORT_MINUTE', 0),

figures/sites.py

@@ -134,7 +134,7 @@ def get_site_for_course(course_id):
             site = None
     else:
         # Operating in single site / standalone mode, return the default site
-        site = Site.objects.get(id=settings.SITE_ID)
+        site = default_site()
     return site
 
 

tests/factories.py

@@ -221,6 +221,20 @@ class Meta:
         2018,2,2, tzinfo=factory.compat.UTC))
 
 
+    @classmethod
+    def from_course_enrollment(cls, course_enrollment, **kwargs):
+        """Contruct a StudentModule  for the given CourseEnrollment
+
+        kwargs provides for additional optional parameters if you need to
+        override the default factory assignment
+        """
+        kwargs.update({
+            'student': course_enrollment.user,
+            'course_id': course_enrollment.course_id,
+            })
+        return cls(**kwargs)
+
+
 if OPENEDX_RELEASE == GINKGO:
     class CourseEnrollmentFactory(DjangoModelFactory):
         class Meta: