Download extra channels in the background

Now that there's a way to serialize background tasks, defer downloading
the extra channels so that the initial download completes faster. Since
the thumbnail tasks depend on this, they need to be generated on the fly
when the channel download completes.

Fixes: #592

kolibri_explore_plugin/collectionviews.py

@@ -51,8 +51,7 @@
 
 PROGRESS_STEPS = {
     "importing": 0.1,
-    "downloading": 0.8,
-    "tagging": 0.9,
+    "downloading": 0.9,
     "completed": 1,
 }
 
@@ -183,16 +182,6 @@ def get_contentthumbnail_tasks(self):
             for channel_id in self.get_channel_ids()
         ]
 
-    def get_extra_contentthumbnail_tasks(self):
-        """Return a serializable object to create thumbnail contentimport tasks
-
-        For all channels featured in other Endless Key content manifests.
-        """
-        return [
-            get_remotecontentimport_task(channel_id, all_thumbnails=True)
-            for channel_id in self.get_extra_channel_ids()
-        ]
-
     def _get_node_ids_for_channel(self, channel_metadata, channel_id):
         """Get node IDs regardless of the version
 
@@ -227,7 +216,6 @@ class DownloadStage(IntEnum):
     IMPORTING_CHANNELS = auto()
     IMPORTING_CONTENT = auto()
     APPLYING_EXTERNAL_TAGS = auto()
-    IMPORTING_EXTRA_CHANNELS = auto()
     COMPLETED = auto()
 
 
@@ -416,7 +404,7 @@ def get_status(self):
                 progress = (
                     PROGRESS_STEPS["downloading"]
                     + (
-                        PROGRESS_STEPS["tagging"]
+                        PROGRESS_STEPS["completed"]
                         - PROGRESS_STEPS["downloading"]
                     )
                     * current_task_number
@@ -425,17 +413,6 @@ def get_status(self):
             else:
                 progress = PROGRESS_STEPS["downloading"]
 
-        elif self._stage == DownloadStage.IMPORTING_EXTRA_CHANNELS:
-            if total_tasks_number > 0:
-                progress = (
-                    PROGRESS_STEPS["tagging"]
-                    + (PROGRESS_STEPS["completed"] - PROGRESS_STEPS["tagging"])
-                    * current_task_number
-                    / total_tasks_number
-                )
-            else:
-                progress = PROGRESS_STEPS["tagging"]
-
         elif self._stage >= DownloadStage.COMPLETED:
             progress = PROGRESS_STEPS["completed"]
 
@@ -490,19 +467,19 @@ def _set_next_stage(self, user):
                 tasks = self._content_manifest.get_contentimport_tasks()
             elif self._stage == DownloadStage.APPLYING_EXTERNAL_TAGS:
                 tasks = self._content_manifest.get_applyexternaltags_tasks()
-            elif self._stage == DownloadStage.IMPORTING_EXTRA_CHANNELS:
-                tasks = self._content_manifest.get_extra_channelimport_tasks()
 
         if self._stage == DownloadStage.COMPLETED:
             logger.info("Download completed!")
 
-            # Download the remaining content thumbnails in the background.
-            # Prioritize the channels from the content manifest.
+            # Download the manifest content thumbnails and the extra channels
+            # in the background.
             thumbnail_tasks = (
                 self._content_manifest.get_contentthumbnail_tasks()
-                + self._content_manifest.get_extra_contentthumbnail_tasks()
             )
-            for task in thumbnail_tasks:
+            extra_channel_tasks = (
+                self._content_manifest.get_extra_channelimport_tasks()
+            )
+            for task in thumbnail_tasks + extra_channel_tasks:
                 BackgroundTask.create_from_task_data(task)
             logger.info("Starting background download tasks")
             enqueue_next_background_task()

kolibri_explore_plugin/jobs.py

@@ -166,5 +166,15 @@ def storage_update_hook(job, orm_job, state=None, **kwargs):
         new_job_id = job_storage.restart_job(bg_task.job_id)
         bg_task.update_job_id(new_job_id)
     elif state == State.COMPLETED:
+        # If the completed task is a channel import, create the
+        # associated thumbnail download task to be run later.
+        if bg_task.func == TaskType.REMOTECHANNELIMPORT:
+            bg_task_params = json.loads(bg_task.params)
+            channel_id = bg_task_params["channel_id"]
+            thumbnail_task_data = get_remotecontentimport_task(
+                channel_id, all_thumbnails=True
+            )
+            BackgroundTask.create_from_task_data(thumbnail_task_data)
+
         # Start the next background task, if any.
         enqueue_next_background_task()