Download: Add a stage to import all channels

After the `APPLYING_EXTERNAL_TAGS` stage, the DownloadManager will enter
a new `IMPORTING_EXTRA_CHANNELS` stage. In this stage, it should
download metadata for any channels which are listed in another content
collection.

Issue #548

kolibri_explore_plugin/collectionviews.py

@@ -48,7 +48,9 @@
 
 PROGRESS_STEPS = {
     "importing": 0.1,
-    "downloading": 0.95,
+    "downloading": 0.8,
+    "tagging": 0.9,
+    "completed": 1,
 }
 
 
@@ -69,6 +71,10 @@ def __init__(self):
         self.available = None
         super().__init__()
 
+    def get_extra_channel_ids(self):
+        all_channel_ids = _get_channel_ids_for_all_content_manifests()
+        return all_channel_ids.difference(self.get_channel_ids())
+
     def read_from_static_collection(self, grade, name, validate=False):
         self.grade = grade
         self.name = name
@@ -112,22 +118,20 @@ def get_channelimport_tasks(self):
 
         For all the channels in this content manifest.
         """
-        tasks = []
+        return [
+            _build_remotechannelimport_task(channel_id)
+            for channel_id in self.get_channel_ids()
+        ]
 
-        for channel_id in self.get_channel_ids():
-            tasks.append(
-                {
-                    "task": "remotechannelimport",
-                    "params": {
-                        "channel_id": channel_id,
-                        # FIXME: The channel_name is needed
-                        # since commit b53d7baa
-                        "channel_name": "foo",
-                    },
-                }
-            )
+    def get_extra_channelimport_tasks(self):
+        """Return a serializable object to create extra channelimport tasks
 
-        return tasks
+        For all channels featured in Endless Key content manifests.
+        """
+        return [
+            _build_remotechannelimport_task(channel_id)
+            for channel_id in self.get_extra_channel_ids()
+        ]
 
     def get_contentimport_tasks(self):
         """Return a serializable object to create contentimport tasks
@@ -227,6 +231,7 @@ class DownloadStage(IntEnum):
     IMPORTING_CHANNELS = auto()
     IMPORTING_CONTENT = auto()
     APPLYING_EXTERNAL_TAGS = auto()
+    IMPORTING_EXTRA_CHANNELS = auto()
     COMPLETED = auto()
 
 
@@ -387,17 +392,6 @@ def get_status(self):
             else:
                 progress = 0
 
-        elif self._stage == DownloadStage.APPLYING_EXTERNAL_TAGS:
-            if total_tasks_number > 0:
-                progress = (
-                    PROGRESS_STEPS["downloading"]
-                    + (1 - PROGRESS_STEPS["downloading"])
-                    * current_task_number
-                    / total_tasks_number
-                )
-            else:
-                progress = 0
-
         elif self._stage == DownloadStage.IMPORTING_CONTENT:
             if self._current_job_id is None:
                 progress = PROGRESS_STEPS["importing"]
@@ -421,8 +415,33 @@ def get_status(self):
                     + progress_per_channel * current_job_factor
                 )
 
+        elif self._stage == DownloadStage.APPLYING_EXTERNAL_TAGS:
+            if total_tasks_number > 0:
+                progress = (
+                    PROGRESS_STEPS["downloading"]
+                    + (
+                        PROGRESS_STEPS["tagging"]
+                        - PROGRESS_STEPS["downloading"]
+                    )
+                    * current_task_number
+                    / total_tasks_number
+                )
+            else:
+                progress = 0
+
+        elif self._stage == DownloadStage.IMPORTING_EXTRA_CHANNELS:
+            if total_tasks_number > 0:
+                progress = (
+                    PROGRESS_STEPS["tagging"]
+                    + (PROGRESS_STEPS["completed"] - PROGRESS_STEPS["tagging"])
+                    * current_task_number
+                    / total_tasks_number
+                )
+            else:
+                progress = 0
+
         elif self._stage == DownloadStage.COMPLETED:
-            progress = 1
+            progress = PROGRESS_STEPS["completed"]
 
         return {
             "stage": self._stage.name,
@@ -476,6 +495,8 @@ def _set_next_stage(self):
                 tasks = self._content_manifest.get_contentimport_tasks()
             elif self._stage == DownloadStage.APPLYING_EXTERNAL_TAGS:
                 tasks = self._content_manifest.get_applyexternaltags_tasks()
+            elif self._stage == DownloadStage.IMPORTING_EXTRA_CHANNELS:
+                tasks = self._content_manifest.get_extra_channelimport_tasks()
 
         self._tasks_pending = tasks
         self._tasks_previously_completed.extend(self._tasks_completed)
@@ -514,6 +535,18 @@ def _call_task(task, user, **params):
     return job_id
 
 
+def _build_remotechannelimport_task(channel_id):
+    return {
+        "task": "remotechannelimport",
+        "params": {
+            "channel_id": channel_id,
+            # FIXME: The channel_name is needed
+            # since commit b53d7baa
+            "channel_name": "foo",
+        },
+    }
+
+
 _content_manifests = []
 _content_manifests_by_grade_name = {}
 _collection_download_manager = CollectionDownloadManager()
@@ -573,6 +606,13 @@ def _get_collections_info_by_grade_name(grade, name):
     }
 
 
+def _get_channel_ids_for_all_content_manifests():
+    channel_ids = set()
+    for content_manifest in _content_manifests:
+        channel_ids.update(content_manifest.get_channel_ids())
+    return channel_ids
+
+
 @api_view(["GET"])
 def get_collection_info(request):
     """Return the collection metadata and availability."""