Merge pull request #2742 from mrpau/hotfix/2271-Export-CSV-data-quali…

…ty-issues

Hotfix/2271 export csv data quality issues

kalite/control_panel/views.py

@@ -2,6 +2,7 @@
 """
 import copy
 import datetime
+import dateutil
 import re
 import os
 from annoying.decorators import render_to, wraps
@@ -13,6 +14,7 @@
 from django.core.exceptions import ValidationError
 from django.core.urlresolvers import reverse
 from django.db.models import Sum, Max
+from django.db.models.query_utils import Q
 from django.http import Http404, HttpResponse, HttpResponseRedirect
 from django.shortcuts import get_object_or_404
 from django.template import RequestContext
@@ -383,15 +385,33 @@ def _get_user_usage_data(users, groups=None, period_start=None, period_end=None,
     login_logs = UserLogSummary.objects.filter(user__in=users)
 
     # filter results
+    login_logs = login_logs.filter(total_seconds__gt=0)
     if period_start:
         exercise_logs = exercise_logs.filter(completion_timestamp__gte=period_start)
         video_logs = video_logs.filter(completion_timestamp__gte=period_start)
-        login_logs = login_logs.filter(start_datetime__gte=period_start)
     if period_end:
+        # MUST: Fix the midnight bug where period end covers up to the prior day only because
+        # period end is datetime(year, month, day, hour=0, minute=0), meaning midnight of previous day.
+        # Example:
+        #   If period_end == '2014-12-01', we cannot include the records dated '2014-12-01 09:30'.
+        #   So to fix this, we change it to '2014-12-01 23:59.999999'.
+        period_end = dateutil.parser.parse(period_end)
+        period_end = period_end + dateutil.relativedelta.relativedelta(days=+1, microseconds=-1)
         exercise_logs = exercise_logs.filter(completion_timestamp__lte=period_end)
         video_logs = video_logs.filter(completion_timestamp__lte=period_end)
-        login_logs = login_logs.filter(total_seconds__gt=0, start_datetime__lte=period_end)
-
+    if period_start and period_end:
+        exercise_logs = exercise_logs.filter(Q(completion_timestamp__gte=period_start) &
+                                             Q(completion_timestamp__lte=period_end))
+
+        q1 = Q(completion_timestamp__isnull=False) & \
+             Q(completion_timestamp__gte=period_start) & \
+             Q(completion_timestamp__lte=period_end)
+        q2 = Q(completion_timestamp__isnull=True)
+        video_logs = video_logs.filter(q1 | q2)
+
+        login_q1 = Q(start_datetime__gte=period_start) & Q(start_datetime__lte=period_end) & \
+                   Q(end_datetime__gte=period_start) & Q(end_datetime__lte=period_end)
+        login_logs = login_logs.filter(login_q1)
     # Force results in a single query
     exercise_logs = list(exercise_logs.values("exercise_id", "user__pk"))
     video_logs = list(video_logs.values("video_id", "user__pk"))

kalite/main/admin.py

@@ -1,27 +1,30 @@
 """
 Manipulating models to expose to admins through the Django admin interface
 """
-from django.conf import settings
 from django.contrib import admin
 
 from .models import *
 
 
 class VideoLogAdmin(admin.ModelAdmin):
-    list_display = ("video_id", "user", "language", "points", "total_seconds_watched", "complete",)
-    list_filter = ("video_id", "user", "language", "complete",)
+    list_display = ("video_id", "user", "language", "points", "total_seconds_watched", "complete",
+                    "completion_timestamp",)
+    list_filter = ("completion_timestamp", "complete", "language", "user", "video_id",)
 admin.site.register(VideoLog, VideoLogAdmin)
 
+
 class ExerciseLogAdmin(admin.ModelAdmin):
     list_display = ("exercise_id", "user", "language", "streak_progress", "completion_timestamp", "complete",)
     list_filter = ("completion_timestamp", "complete", "language", "exercise_id", "user",)
 admin.site.register(ExerciseLog, ExerciseLogAdmin)
 
+
 class UserLogAdmin(admin.ModelAdmin):
     pass
 if UserLog.is_enabled():  # only enable admin if the feature is enabled.
     admin.site.register(UserLog, UserLogAdmin)
 
+
 class UserLogSummaryAdmin(admin.ModelAdmin):
     pass
 if UserLog.is_enabled() or settings.CENTRAL_SERVER:  # only enable admin if the feature is enabled.

kalite/main/settings.py

@@ -28,4 +28,4 @@
 # Used for user logs.  By default, completely off.
 #   NOTE: None means no limit (infinite)
 USER_LOG_MAX_RECORDS_PER_USER = getattr(local_settings, "USER_LOG_MAX_RECORDS_PER_USER", 1)
-USER_LOG_SUMMARY_FREQUENCY = getattr(local_settings, "USER_LOG_SUMMARY_FREQUENCY", (1,"months"))
+USER_LOG_SUMMARY_FREQUENCY = getattr(local_settings, "USER_LOG_SUMMARY_FREQUENCY", (1,"day"))