Add background job queue and restart failed jobs during startup

In order to differentiate foreground tasks from background tasks, a
separate background job queue is added. When the plugin is loaded, it
will immediately restart any failed background tasks. No frontend
integration is added here, but the idea is that the frontend will query
both the foreground and background queues for completion and update the
UI as needed.

kolibri_explore_plugin/__init__.py

@@ -1 +1,3 @@
 __version__ = "6.16.0"
+
+default_app_config = "kolibri_explore_plugin.apps_config.ExploreConfig"

kolibri_explore_plugin/apps_config.py

@@ -0,0 +1,11 @@
+from django.apps import AppConfig
+
+
+class ExploreConfig(AppConfig):
+    name = "kolibri_explore_plugin"
+    label = "explore"
+
+    def ready(self):
+        from .tasks import restart_failed_background_jobs
+
+        restart_failed_background_jobs()

kolibri_explore_plugin/tasks.py

@@ -1,3 +1,5 @@
+import logging
+
 from django.core.management import call_command
 from kolibri.core.content.tasks import get_status as get_content_task_status
 from kolibri.core.content.tasks import RemoteChannelResourcesImportValidator
@@ -13,14 +15,19 @@
 )
 from kolibri.core.serializers import HexOnlyUUIDField
 from kolibri.core.tasks.decorators import register_task
+from kolibri.core.tasks.job import State
+from kolibri.core.tasks.main import job_storage
 from kolibri.core.tasks.permissions import CanManageContent
 from kolibri.core.tasks.validation import JobValidator
 from rest_framework.serializers import CharField
 from rest_framework.serializers import ListField
 
 from .vendor.file_transfer import FileDownload
 
+logger = logging.getLogger(__name__)
+
 QUEUE = "content"
+BACKGROUND_QUEUE = "explore-bg"
 
 
 class ExternalTagsJobValidator(JobValidator):
@@ -111,3 +118,12 @@ def remotecontentimport(
         all_thumbnails=all_thumbnails,
     )
     manager.run()
+
+
+def restart_failed_background_jobs():
+    for job in job_storage.get_all_jobs(queue=BACKGROUND_QUEUE):
+        if job.state == State.FAILED:
+            logger.info(
+                f"Restarting failed background {job.func} job {job.job_id}"
+            )
+            job_storage.restart_job(job.job_id)