From 6a22ea9de290176932fc8e0b0a94381e61467804 Mon Sep 17 00:00:00 2001
From: Dan Nicholson <dbn@endlessos.org>
Date: Mon, 21 Aug 2023 12:46:38 -0600
Subject: [PATCH] Download extra channels in the background

Now that there's a way to serialize background tasks, defer downloading
the extra channels so that the initial download completes faster. Since
the thumbnail tasks depend on this, they need to be generated on the fly
when the channel download completes.

Fixes: #592
---
 kolibri_explore_plugin/collectionviews.py | 39 +++++------------------
 kolibri_explore_plugin/jobs.py            | 10 ++++++
 2 files changed, 18 insertions(+), 31 deletions(-)

diff --git a/kolibri_explore_plugin/collectionviews.py b/kolibri_explore_plugin/collectionviews.py
index 91166bbcd..0faf51c7b 100644
--- a/kolibri_explore_plugin/collectionviews.py
+++ b/kolibri_explore_plugin/collectionviews.py
@@ -51,8 +51,7 @@
 
 PROGRESS_STEPS = {
     "importing": 0.1,
-    "downloading": 0.8,
-    "tagging": 0.9,
+    "downloading": 0.9,
     "completed": 1,
 }
 
@@ -183,16 +182,6 @@ def get_contentthumbnail_tasks(self):
             for channel_id in self.get_channel_ids()
         ]
 
-    def get_extra_contentthumbnail_tasks(self):
-        """Return a serializable object to create thumbnail contentimport tasks
-
-        For all channels featured in other Endless Key content manifests.
-        """
-        return [
-            get_remotecontentimport_task(channel_id, all_thumbnails=True)
-            for channel_id in self.get_extra_channel_ids()
-        ]
-
     def _get_node_ids_for_channel(self, channel_metadata, channel_id):
         """Get node IDs regardless of the version
 
@@ -227,7 +216,6 @@ class DownloadStage(IntEnum):
     IMPORTING_CHANNELS = auto()
     IMPORTING_CONTENT = auto()
     APPLYING_EXTERNAL_TAGS = auto()
-    IMPORTING_EXTRA_CHANNELS = auto()
     COMPLETED = auto()
 
 
@@ -416,7 +404,7 @@ def get_status(self):
                 progress = (
                     PROGRESS_STEPS["downloading"]
                     + (
-                        PROGRESS_STEPS["tagging"]
+                        PROGRESS_STEPS["completed"]
                         - PROGRESS_STEPS["downloading"]
                     )
                     * current_task_number
@@ -425,17 +413,6 @@ def get_status(self):
             else:
                 progress = PROGRESS_STEPS["downloading"]
 
-        elif self._stage == DownloadStage.IMPORTING_EXTRA_CHANNELS:
-            if total_tasks_number > 0:
-                progress = (
-                    PROGRESS_STEPS["tagging"]
-                    + (PROGRESS_STEPS["completed"] - PROGRESS_STEPS["tagging"])
-                    * current_task_number
-                    / total_tasks_number
-                )
-            else:
-                progress = PROGRESS_STEPS["tagging"]
-
         elif self._stage >= DownloadStage.COMPLETED:
             progress = PROGRESS_STEPS["completed"]
 
@@ -490,19 +467,19 @@ def _set_next_stage(self, user):
                 tasks = self._content_manifest.get_contentimport_tasks()
             elif self._stage == DownloadStage.APPLYING_EXTERNAL_TAGS:
                 tasks = self._content_manifest.get_applyexternaltags_tasks()
-            elif self._stage == DownloadStage.IMPORTING_EXTRA_CHANNELS:
-                tasks = self._content_manifest.get_extra_channelimport_tasks()
 
         if self._stage == DownloadStage.COMPLETED:
             logger.info("Download completed!")
 
-            # Download the remaining content thumbnails in the background.
-            # Prioritize the channels from the content manifest.
+            # Download the manifest content thumbnails and the extra channels
+            # in the background.
             thumbnail_tasks = (
                 self._content_manifest.get_contentthumbnail_tasks()
-                + self._content_manifest.get_extra_contentthumbnail_tasks()
             )
-            for task in thumbnail_tasks:
+            extra_channel_tasks = (
+                self._content_manifest.get_extra_channelimport_tasks()
+            )
+            for task in thumbnail_tasks + extra_channel_tasks:
                 BackgroundTask.create_from_task_data(task)
             logger.info("Starting background download tasks")
             enqueue_next_background_task()
diff --git a/kolibri_explore_plugin/jobs.py b/kolibri_explore_plugin/jobs.py
index a88679a90..a6d3d9e4d 100644
--- a/kolibri_explore_plugin/jobs.py
+++ b/kolibri_explore_plugin/jobs.py
@@ -166,5 +166,15 @@ def storage_update_hook(job, orm_job, state=None, **kwargs):
         new_job_id = job_storage.restart_job(bg_task.job_id)
         bg_task.update_job_id(new_job_id)
     elif state == State.COMPLETED:
+        # If the completed task is a channel import, create the
+        # associated thumbnail download task to be run later.
+        if bg_task.func == TaskType.REMOTECHANNELIMPORT:
+            bg_task_params = json.loads(bg_task.params)
+            channel_id = bg_task_params["channel_id"]
+            thumbnail_task_data = get_remotecontentimport_task(
+                channel_id, all_thumbnails=True
+            )
+            BackgroundTask.create_from_task_data(thumbnail_task_data)
+
         # Start the next background task, if any.
         enqueue_next_background_task()