Merge pull request #3274 from learningequality/hotfixes

Merge down hotfixes to unstable

contentcuration/contentcuration/frontend/shared/data/index.js

@@ -48,7 +48,7 @@ function runElection() {
   elector.awaitLeadership().then(startSyncing);
   elector.onduplicate = () => {
     stopSyncing();
-    elector.die.then(runElection);
+    elector.die().then(runElection);
   };
 }
 

contentcuration/contentcuration/management/commands/garbage_collect.py

@@ -4,6 +4,8 @@
 tree" (i.e. `settings.ORPHANAGE_ROOT_ID`). Also delete the associated Files in the
 database and in object storage.
 """
+import logging as logmodule
+
 from django.core.management.base import BaseCommand
 
 from contentcuration.utils.garbage_collect import clean_up_contentnodes
@@ -12,6 +14,10 @@
 from contentcuration.utils.garbage_collect import clean_up_tasks
 
 
+logmodule.basicConfig(level=logmodule.INFO)
+logging = logmodule.getLogger('command')
+
+
 class Command(BaseCommand):
 
     def handle(self, *args, **options):
@@ -21,7 +27,11 @@ def handle(self, *args, **options):
 
         # clean up contentnodes, files and file objects on storage that are associated
         # with the orphan tree
+        logging.info("Cleaning up contentnodes from the orphan tree")
         clean_up_contentnodes()
+        logging.info("Cleaning up deleted chef nodes")
         clean_up_deleted_chefs()
+        logging.info("Cleaning up feature flags")
         clean_up_feature_flags()
+        logging.info("Cleaning up tasks")
         clean_up_tasks()

contentcuration/contentcuration/settings.py

@@ -14,11 +14,11 @@
 import os
 import re
 import sys
-from datetime import datetime
 from datetime import timedelta
 from tempfile import gettempdir
 
 import pycountry
+from django.utils.timezone import now
 
 from contentcuration.utils.incidents import INCIDENTS
 from contentcuration.utils.secretmanagement import get_secret
@@ -363,7 +363,7 @@ def gettext(s):
 # When cleaning up orphan nodes, only clean up any that have been last modified
 # since this date
 # our default threshold is two weeks ago
-TWO_WEEKS_AGO = datetime.now() - timedelta(days=14)
+TWO_WEEKS_AGO = now() - timedelta(days=14)
 ORPHAN_DATE_CLEAN_UP_THRESHOLD = TWO_WEEKS_AGO
 
 # CLOUD STORAGE SETTINGS

contentcuration/contentcuration/tasks.py

@@ -67,6 +67,8 @@ def delete_node_task(
                     pass
                 else:
                     raise
+            except ContentNode.DoesNotExist:
+                deleted = True
     except Exception as e:
         self.report_exception(e)
 

contentcuration/contentcuration/utils/garbage_collect.py

@@ -3,12 +3,15 @@
 Studio garbage collection utilities. Clean up all these old, unused records!
 """
 import datetime
+import logging
 
 from celery import states
 from django.conf import settings
 from django.db.models.expressions import CombinedExpression
 from django.db.models.expressions import F
 from django.db.models.expressions import Value
+from django.db.models.signals import post_delete
+from django.utils.timezone import now
 from le_utils.constants import content_kinds
 
 from contentcuration.constants import feature_flags
@@ -19,6 +22,21 @@
 from contentcuration.models import User
 
 
+class DisablePostDeleteSignal(object):
+    """
+    Helper that disables the post_delete signal temporarily when deleting, so Morango doesn't
+    create DeletedModels objects for what we're deleting
+    """
+
+    def __enter__(self):
+        self.receivers = post_delete.receivers
+        post_delete.receivers = []
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        post_delete.receivers = self.receivers
+        self.receivers = None
+
+
 def get_deleted_chefs_root():
     deleted_chefs_node, _new = ContentNode.objects.get_or_create(pk=settings.DELETED_CHEFS_ROOT_ID, kind_id=content_kinds.TOPIC)
     return deleted_chefs_node
@@ -37,12 +55,14 @@ def clean_up_deleted_chefs():
 
     # don't delete files until we can ensure files are not referenced anywhere.
     # disable mptt updates as they are disabled when we insert nodes into this tree
-    with ContentNode.objects.disable_mptt_updates():
-        for node in nodes_to_clean_up:
-            node.delete()
-
-    if ContentNode.objects.filter(parent=deleted_chefs_node).exists():
-        raise AssertionError
+    with ContentNode.objects.disable_mptt_updates(), DisablePostDeleteSignal():
+        for i, node in enumerate(nodes_to_clean_up):
+            try:
+                node.delete()
+            except ContentNode.DoesNotExist:
+                # If it doesn't exist, job done!
+                pass
+            logging.info("Deleted {} node(s) from the deleted chef tree".format(i + 1))
 
 
 def clean_up_contentnodes(delete_older_than=settings.ORPHAN_DATE_CLEAN_UP_THRESHOLD):
@@ -60,10 +80,16 @@ def clean_up_contentnodes(delete_older_than=settings.ORPHAN_DATE_CLEAN_UP_THRESH
     )
 
     # delete all files first
-    clean_up_files(nodes_to_clean_up)
+    with DisablePostDeleteSignal():
+        clean_up_files(nodes_to_clean_up)
 
     # Use _raw_delete for fast bulk deletions
-    nodes_to_clean_up.delete()
+    try:
+        with DisablePostDeleteSignal():
+            count, _ = nodes_to_clean_up.delete()
+        logging.info("Deleted {} node(s) from the orphanage tree".format(count))
+    except ContentNode.DoesNotExist:
+        pass
 
 
 def clean_up_files(contentnode_ids):
@@ -114,4 +140,6 @@ def clean_up_tasks():
     """
     Removes completed tasks that are older than a week
     """
-    Task.objects.filter(created__lt=datetime.datetime.now() - datetime.timedelta(days=7), status=states.SUCCESS).delete()
+    with DisablePostDeleteSignal():
+        count, _ = Task.objects.filter(created__lt=now() - datetime.timedelta(days=7), status=states.SUCCESS).delete()
+    logging.info("Deleted {} successful task(s) from the task queue".format(count))

k8s/templates/garbage-collect-cronjob.yaml

@@ -52,9 +52,9 @@ spec:
           - name: app
             image: {{ .Values.studioApp.imageName }}
             command:
-            - bash
-            - -c
-            - 'echo "python contentcuration/manage.py garbage_collect"'
+            - python
+            - contentcuration/manage.py
+            - garbage_collect
             env:
             - name: DJANGO_SETTINGS_MODULE
               value: contentcuration.production_settings