Merge pull request #144 from stanfordnmbl/feat/db_clean

Periodic database cleaning

mcserver/settings.py

@@ -248,6 +248,9 @@ def strip_sensitive_data(event, hint):
     'TRASHED_OBJECTS_CLEANUP_DAYS', default=30, cast=int)  # 30 days by default
 ARCHIVE_CLEANUP_DAYS = config('ARCHIVE_CLEANUP_DAYS', default=4, cast=int)
 
+# An option to disable the cleanup of the unused data for debugging purposes
+CLEANUP_UNUSED_DATA = config('CLEANUP_UNUSED_DATA', default=True, cast=bool)
+
 CELERY_BROKER_URL = REDIS_URL
 CELERY_RESULT_BACKEND = REDIS_URL
 CELERY_BEAT_SCHEDULER = 'redbeat.RedBeatScheduler'
@@ -268,8 +271,16 @@ def strip_sensitive_data(event, hint):
         'task': 'mcserver.tasks.cleanup_archives',
         'schedule': crontab(hour=0, minute=0)
     },
-    'cleanup_archives': {
+    'cleanup_pingdom_sessions': {
         'task': 'mcserver.tasks.delete_pingdom_sessions',
         'schedule': crontab(hour='*', minute=0)
-    }
+    },
+    'cleanup_unused_sessions': {
+        'task': 'mcserver.tasks.cleanup_unused_sessions',
+        'schedule': crontab(hour='*/4', minute=10)
+    },
+    'cleanup_stuck_trials': {
+        'task': 'mcserver.tasks.cleanup_stuck_trials',
+        'schedule': crontab(hour='*/4', minute=30)
+    },
 }

mcserver/tasks.py

@@ -156,3 +156,33 @@ def invoke_aws_lambda_function(self, user_id, function_id, data):
                 template=dashboard_template
             )
 
+@shared_task
+def cleanup_unused_sessions():
+    """ This task deletes all Session's that are not used.
+    """
+    from .models import Session
+
+    if settings.CLEANUP_UNUSED_DATA:
+        now = timezone.now()
+        # Limit to 50 sessions to avoid long running queries
+        old_sessions = Session.objects.filter(updated_at__lt=now-timedelta(days=7))[:100]
+        for session in old_sessions:
+            neutrals = session.trial_set.filter(name__exact='neutral')
+            if not neutrals.exists():
+                session.delete()
+
+
+@shared_task
+def cleanup_stuck_trials():
+    """ This task deletes all Trial's that are not used.
+    """
+    from .models import Trial
+
+    if settings.CLEANUP_UNUSED_DATA:
+        now = timezone.now()
+        # Limit to 50 trials to avoid long running queries
+        stuck_trials = Trial.objects.filter(
+            created_at__lt=now-timedelta(days=1),
+            status='recording',
+        )
+        stuck_trials.delete()