Merge pull request #10780 from dbnicholson/all-thumbnails

Allow including all thumbnails in import
learningequality · Jun 9, 2023 · 0e2dc3a · 0e2dc3a
2 parents be42ec5 + 7a1a6cf
commit 0e2dc3a
Show file tree

Hide file tree

Showing 9 changed files with 351 additions and 21 deletions.
diff --git a/kolibri/core/content/fixtures/content_test.json b/kolibri/core/content/fixtures/content_test.json
@@ -342,6 +342,71 @@
     "pk": "8ad3fffedf144cba9492e16daec1e39b",
     "model": "content.file"
   },
+  {
+    "fields": {
+      "contentnode": "2e8bac07947855369fe2d77642dfc870",
+      "lang": null,
+      "local_file": "2318e5a9d6a24ae8f96e9110006e0c53",
+      "preset": "topic_thumbnail",
+      "priority": null,
+      "supplementary": false,
+      "thumbnail": true
+    },
+    "model": "content.file",
+    "pk": "2318e5a9d6a24ae8f96e9110006e0c53"
+  },
+  {
+    "fields": {
+      "contentnode": "42a941fb77c2576e8f6b294cde4c3b0c",
+      "lang": null,
+      "local_file": "37c5c250fbc66e597ae7d604846e9df2",
+      "preset": "video_thumbnail",
+      "priority": null,
+      "supplementary": false,
+      "thumbnail": true
+    },
+    "model": "content.file",
+    "pk": "37c5c250fbc66e597ae7d604846e9df2"
+  },
+  {
+    "fields": {
+      "contentnode": "2b6926ed22025518a8b9da91745b51d3",
+      "lang": null,
+      "local_file": "2cea0feba5f930c81661c5c759943964",
+      "preset": "exercise_thumbnail",
+      "priority": null,
+      "supplementary": false,
+      "thumbnail": true
+    },
+    "model": "content.file",
+    "pk": "2cea0feba5f930c81661c5c759943964"
+  },
+  {
+    "fields": {
+      "contentnode": "32a941fb77c2576e8f6b294cde4c3b0c",
+      "lang": null,
+      "local_file": "5437c68903de934521128d7656a3b572",
+      "preset": "video_thumbnail",
+      "priority": null,
+      "supplementary": false,
+      "thumbnail": true
+    },
+    "model": "content.file",
+    "pk": "5437c68903de934521128d7656a3b572"
+  },
+  {
+    "fields": {
+      "contentnode": "4d0c890de9b65d6880ccfa527800e0f4",
+      "lang": null,
+      "local_file": "c6f26814b067da30e1cb6239512dc1da",
+      "preset": "document_thumbnail",
+      "priority": null,
+      "supplementary": false,
+      "thumbnail": true
+    },
+    "model": "content.file",
+    "pk": "c6f26814b067da30e1cb6239512dc1da"
+  },
   {
     "fields": {
       "file_size": null,
@@ -387,6 +452,51 @@
     "pk": "8ad3fffedf144cba9492e16daec1e39a",
     "model": "content.localfile"
   },
+  {
+    "fields": {
+      "available": true,
+      "extension": "png",
+      "file_size": 1
+    },
+    "model": "content.localfile",
+    "pk": "2318e5a9d6a24ae8f96e9110006e0c53"
+  },
+  {
+    "fields": {
+      "available": true,
+      "extension": "png",
+      "file_size": 1
+    },
+    "model": "content.localfile",
+    "pk": "37c5c250fbc66e597ae7d604846e9df2"
+  },
+  {
+    "fields": {
+      "available": true,
+      "extension": "jpeg",
+      "file_size": 1
+    },
+    "model": "content.localfile",
+    "pk": "2cea0feba5f930c81661c5c759943964"
+  },
+  {
+    "fields": {
+      "available": true,
+      "extension": "jpeg",
+      "file_size": 1
+    },
+    "model": "content.localfile",
+    "pk": "5437c68903de934521128d7656a3b572"
+  },
+  {
+    "fields": {
+      "available": true,
+      "extension": "png",
+      "file_size": 1
+    },
+    "model": "content.localfile",
+    "pk": "c6f26814b067da30e1cb6239512dc1da"
+  },
   {
     "fields": {
       "name": "testing",

diff --git a/kolibri/core/content/management/commands/importcontent.py b/kolibri/core/content/management/commands/importcontent.py
@@ -90,6 +90,14 @@ def add_arguments(self, parser):
             help="Import all content, not just that which this Kolibri instance can render",
         )
 
+        parser.add_argument(
+            "--all-thumbnails",
+            action="store_true",
+            default=False,
+            dest="all_thumbnails",
+            help="Import thumbnails for all content nodes regardless of included or excluded node IDs",
+        )
+
         parser.add_argument(
             "--import_updates",
             action="store_true",
@@ -177,6 +185,7 @@ def download_content(
         baseurl=None,
         peer_id=None,
         renderable_only=True,
+        all_thumbnails=False,
         import_updates=False,
         fail_on_error=False,
         timeout=transfer.Transfer.DEFAULT_TIMEOUT,
@@ -199,6 +208,7 @@ def download_content(
                 baseurl=baseurl,
                 peer_id=peer_id,
                 renderable_only=renderable_only,
+                all_thumbnails=all_thumbnails,
                 fail_on_error=fail_on_error,
                 timeout=timeout,
                 content_dir=content_dir,
@@ -210,6 +220,7 @@ def download_content(
             baseurl=baseurl,
             peer_id=peer_id,
             renderable_only=renderable_only,
+            all_thumbnails=all_thumbnails,
             fail_on_error=fail_on_error,
             timeout=timeout,
             content_dir=content_dir,
@@ -225,6 +236,7 @@ def copy_content(
         node_ids=None,
         exclude_node_ids=None,
         renderable_only=True,
+        all_thumbnails=False,
         import_updates=False,
         fail_on_error=False,
         content_dir=None,
@@ -242,6 +254,7 @@ def copy_content(
                     path=path,
                     drive_id=drive_id,
                     renderable_only=renderable_only,
+                    all_thumbnails=all_thumbnails,
                     fail_on_error=fail_on_error,
                     content_dir=content_dir,
                 )
@@ -251,6 +264,7 @@ def copy_content(
                     path=path,
                     drive_id=drive_id,
                     renderable_only=renderable_only,
+                    all_thumbnails=all_thumbnails,
                     fail_on_error=fail_on_error,
                     content_dir=content_dir,
                 )
@@ -261,6 +275,7 @@ def copy_content(
             node_ids=node_ids,
             exclude_node_ids=exclude_node_ids,
             renderable_only=renderable_only,
+            all_thumbnails=all_thumbnails,
             fail_on_error=fail_on_error,
             content_dir=content_dir,
         )
@@ -293,6 +308,7 @@ def handle(self, *args, **options):
                 baseurl=options["baseurl"],
                 peer_id=options["peer_id"],
                 renderable_only=options["renderable_only"],
+                all_thumbnails=options["all_thumbnails"],
                 import_updates=options["import_updates"],
                 fail_on_error=options["fail_on_error"],
                 timeout=options["timeout"],
@@ -313,6 +329,7 @@ def handle(self, *args, **options):
                 node_ids=options["node_ids"],
                 exclude_node_ids=options["exclude_node_ids"],
                 renderable_only=options["renderable_only"],
+                all_thumbnails=options["all_thumbnails"],
                 import_updates=options["import_updates"],
                 fail_on_error=options["fail_on_error"],
                 content_dir=options["content_dir"],

diff --git a/kolibri/core/content/tasks.py b/kolibri/core/content/tasks.py
@@ -92,13 +92,15 @@ class ChannelResourcesImportValidator(ChannelResourcesValidator):
     update = serializers.BooleanField(default=False)
     fail_on_error = serializers.BooleanField(default=False)
     new_version = serializers.IntegerField(required=False)
+    all_thumbnails = serializers.BooleanField(default=False)
 
     def validate(self, data):
         job_data = super(ChannelResourcesImportValidator, self).validate(data)
         job_data["kwargs"].update(
             {
                 "update": data.get("update"),
                 "fail_on_error": data.get("fail_on_error"),
+                "all_thumbnails": data.get("all_thumbnails"),
             }
         )
         if data.get("new_version"):
@@ -147,6 +149,7 @@ def diskcontentimport(
     node_ids=None,
     exclude_node_ids=None,
     fail_on_error=False,
+    all_thumbnails=False,
 ):
     manager_class = (
         DiskChannelUpdateManager if update else DiskChannelResourceImportManager
@@ -159,6 +162,7 @@ def diskcontentimport(
         node_ids=node_ids,
         exclude_node_ids=exclude_node_ids,
         fail_on_error=fail_on_error,
+        all_thumbnails=all_thumbnails,
     )
     manager.run()
 
@@ -233,6 +237,7 @@ def remotecontentimport(
     exclude_node_ids=None,
     update=False,
     fail_on_error=False,
+    all_thumbnails=False,
 ):
     manager_class = (
         RemoteChannelUpdateManager if update else RemoteChannelResourceImportManager
@@ -244,6 +249,7 @@ def remotecontentimport(
         node_ids=node_ids,
         exclude_node_ids=exclude_node_ids,
         fail_on_error=fail_on_error,
+        all_thumbnails=all_thumbnails,
     )
     manager.run()
 
@@ -434,6 +440,7 @@ def remoteimport(
     exclude_node_ids=None,
     update=False,
     fail_on_error=False,
+    all_thumbnails=False,
 ):
     call_command(
         "importchannel",
@@ -457,6 +464,7 @@ def remoteimport(
         node_ids=node_ids,
         exclude_node_ids=exclude_node_ids,
         fail_on_error=fail_on_error,
+        all_thumbnails=all_thumbnails,
     )
     manager.run()
 
@@ -471,7 +479,12 @@ def remoteimport(
     status_fn=get_status,
 )
 def diskimport(
-    channel_id, drive_id, update=False, node_ids=None, exclude_node_ids=None
+    channel_id,
+    drive_id,
+    update=False,
+    node_ids=None,
+    exclude_node_ids=None,
+    all_thumbnails=False,
 ):
     drive = get_mounted_drive_by_id(drive_id)
     directory = drive.datafolder
@@ -498,6 +511,7 @@ def diskimport(
         node_ids=node_ids,
         exclude_node_ids=exclude_node_ids,
         import_updates=update,
+        all_thumbnails=all_thumbnails,
     )
     manager.run()
 

diff --git a/kolibri/core/content/test/test_annotation.py b/kolibri/core/content/test/test_annotation.py
@@ -823,12 +823,12 @@ def test_set_all_files_all_exist(self, path_mock):
 
     @patch(
         "kolibri.core.content.utils.annotation.get_content_storage_file_path",
-        side_effect=[mock_content_file[1]] * 2 + [""] * 3,
+        side_effect=[mock_content_file[1]] * 2 + [""] * 8,
     )
     def test_set_all_files_two_exist(self, path_mock):
         set_local_file_availability_from_disk()
         self.assertEqual(LocalFile.objects.filter(available=True).count(), 2)
-        self.assertEqual(LocalFile.objects.exclude(available=True).count(), 3)
+        self.assertEqual(LocalFile.objects.exclude(available=True).count(), 8)
 
     def test_set_bad_filenames(self):
         local_files = list(LocalFile.objects.all())

diff --git a/kolibri/core/content/test/test_content_app.py b/kolibri/core/content/test/test_content_app.py
@@ -1221,7 +1221,7 @@ def test_channelmetadata_has_exercises_filter(self):
 
     def test_file_list(self):
         response = self.client.get(reverse("kolibri:core:file-list"))
-        self.assertEqual(len(response.data), 5)
+        self.assertEqual(len(response.data), 10)
 
     def test_file_retrieve(self):
         response = self.client.get(