Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow including all thumbnails in import #10780

Merged
merged 5 commits into from
Jun 9, 2023
Merged

Allow including all thumbnails in import #10780

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c9af043
Add thumbnails to content_test fixture
dbnicholson May 31, 2023
6dd6d5c
Allow requesting all thumbnails in import/export data
dbnicholson May 31, 2023
5a95606
Wire up all_thumbnails option to import managers
dbnicholson Jun 1, 2023
77c1380
Wire up all_thumbnails option to import tasks
dbnicholson Jun 1, 2023
7a1a6cf
Wire up all_thumbnails option to importcontent management command
dbnicholson Jun 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 110 additions & 0 deletions kolibri/core/content/fixtures/content_test.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,71 @@
"pk": "8ad3fffedf144cba9492e16daec1e39b",
"model": "content.file"
},
{
"fields": {
"contentnode": "2e8bac07947855369fe2d77642dfc870",
"lang": null,
"local_file": "2318e5a9d6a24ae8f96e9110006e0c53",
"preset": "topic_thumbnail",
"priority": null,
"supplementary": false,
"thumbnail": true
},
"model": "content.file",
"pk": "2318e5a9d6a24ae8f96e9110006e0c53"
},
{
"fields": {
"contentnode": "42a941fb77c2576e8f6b294cde4c3b0c",
"lang": null,
"local_file": "37c5c250fbc66e597ae7d604846e9df2",
"preset": "video_thumbnail",
"priority": null,
"supplementary": false,
"thumbnail": true
},
"model": "content.file",
"pk": "37c5c250fbc66e597ae7d604846e9df2"
},
{
"fields": {
"contentnode": "2b6926ed22025518a8b9da91745b51d3",
"lang": null,
"local_file": "2cea0feba5f930c81661c5c759943964",
"preset": "exercise_thumbnail",
"priority": null,
"supplementary": false,
"thumbnail": true
},
"model": "content.file",
"pk": "2cea0feba5f930c81661c5c759943964"
},
{
"fields": {
"contentnode": "32a941fb77c2576e8f6b294cde4c3b0c",
"lang": null,
"local_file": "5437c68903de934521128d7656a3b572",
"preset": "video_thumbnail",
"priority": null,
"supplementary": false,
"thumbnail": true
},
"model": "content.file",
"pk": "5437c68903de934521128d7656a3b572"
},
{
"fields": {
"contentnode": "4d0c890de9b65d6880ccfa527800e0f4",
"lang": null,
"local_file": "c6f26814b067da30e1cb6239512dc1da",
"preset": "document_thumbnail",
"priority": null,
"supplementary": false,
"thumbnail": true
},
"model": "content.file",
"pk": "c6f26814b067da30e1cb6239512dc1da"
},
{
"fields": {
"file_size": null,
Expand Down Expand Up @@ -387,6 +452,51 @@
"pk": "8ad3fffedf144cba9492e16daec1e39a",
"model": "content.localfile"
},
{
"fields": {
"available": true,
"extension": "png",
"file_size": 1
},
"model": "content.localfile",
"pk": "2318e5a9d6a24ae8f96e9110006e0c53"
},
{
"fields": {
"available": true,
"extension": "png",
"file_size": 1
},
"model": "content.localfile",
"pk": "37c5c250fbc66e597ae7d604846e9df2"
},
{
"fields": {
"available": true,
"extension": "jpeg",
"file_size": 1
},
"model": "content.localfile",
"pk": "2cea0feba5f930c81661c5c759943964"
},
{
"fields": {
"available": true,
"extension": "jpeg",
"file_size": 1
},
"model": "content.localfile",
"pk": "5437c68903de934521128d7656a3b572"
},
{
"fields": {
"available": true,
"extension": "png",
"file_size": 1
},
"model": "content.localfile",
"pk": "c6f26814b067da30e1cb6239512dc1da"
},
{
"fields": {
"name": "testing",
Expand Down
17 changes: 17 additions & 0 deletions kolibri/core/content/management/commands/importcontent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,14 @@ def add_arguments(self, parser):
help="Import all content, not just that which this Kolibri instance can render",
)

parser.add_argument(
"--all-thumbnails",
action="store_true",
default=False,
dest="all_thumbnails",
help="Import thumbnails for all content nodes regardless of included or excluded node IDs",
)

parser.add_argument(
"--import_updates",
action="store_true",
Expand Down Expand Up @@ -177,6 +185,7 @@ def download_content(
baseurl=None,
peer_id=None,
renderable_only=True,
all_thumbnails=False,
import_updates=False,
fail_on_error=False,
timeout=transfer.Transfer.DEFAULT_TIMEOUT,
Expand All @@ -199,6 +208,7 @@ def download_content(
baseurl=baseurl,
peer_id=peer_id,
renderable_only=renderable_only,
all_thumbnails=all_thumbnails,
fail_on_error=fail_on_error,
timeout=timeout,
content_dir=content_dir,
Expand All @@ -210,6 +220,7 @@ def download_content(
baseurl=baseurl,
peer_id=peer_id,
renderable_only=renderable_only,
all_thumbnails=all_thumbnails,
fail_on_error=fail_on_error,
timeout=timeout,
content_dir=content_dir,
Expand All @@ -225,6 +236,7 @@ def copy_content(
node_ids=None,
exclude_node_ids=None,
renderable_only=True,
all_thumbnails=False,
import_updates=False,
fail_on_error=False,
content_dir=None,
Expand All @@ -242,6 +254,7 @@ def copy_content(
path=path,
drive_id=drive_id,
renderable_only=renderable_only,
all_thumbnails=all_thumbnails,
fail_on_error=fail_on_error,
content_dir=content_dir,
)
Expand All @@ -251,6 +264,7 @@ def copy_content(
path=path,
drive_id=drive_id,
renderable_only=renderable_only,
all_thumbnails=all_thumbnails,
fail_on_error=fail_on_error,
content_dir=content_dir,
)
Expand All @@ -261,6 +275,7 @@ def copy_content(
node_ids=node_ids,
exclude_node_ids=exclude_node_ids,
renderable_only=renderable_only,
all_thumbnails=all_thumbnails,
fail_on_error=fail_on_error,
content_dir=content_dir,
)
Expand Down Expand Up @@ -293,6 +308,7 @@ def handle(self, *args, **options):
baseurl=options["baseurl"],
peer_id=options["peer_id"],
renderable_only=options["renderable_only"],
all_thumbnails=options["all_thumbnails"],
import_updates=options["import_updates"],
fail_on_error=options["fail_on_error"],
timeout=options["timeout"],
Expand All @@ -313,6 +329,7 @@ def handle(self, *args, **options):
node_ids=options["node_ids"],
exclude_node_ids=options["exclude_node_ids"],
renderable_only=options["renderable_only"],
all_thumbnails=options["all_thumbnails"],
import_updates=options["import_updates"],
fail_on_error=options["fail_on_error"],
content_dir=options["content_dir"],
Expand Down
16 changes: 15 additions & 1 deletion kolibri/core/content/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,13 +92,15 @@ class ChannelResourcesImportValidator(ChannelResourcesValidator):
update = serializers.BooleanField(default=False)
fail_on_error = serializers.BooleanField(default=False)
new_version = serializers.IntegerField(required=False)
all_thumbnails = serializers.BooleanField(default=False)

def validate(self, data):
job_data = super(ChannelResourcesImportValidator, self).validate(data)
job_data["kwargs"].update(
{
"update": data.get("update"),
"fail_on_error": data.get("fail_on_error"),
"all_thumbnails": data.get("all_thumbnails"),
}
)
if data.get("new_version"):
Expand Down Expand Up @@ -147,6 +149,7 @@ def diskcontentimport(
node_ids=None,
exclude_node_ids=None,
fail_on_error=False,
all_thumbnails=False,
):
manager_class = (
DiskChannelUpdateManager if update else DiskChannelResourceImportManager
Expand All @@ -159,6 +162,7 @@ def diskcontentimport(
node_ids=node_ids,
exclude_node_ids=exclude_node_ids,
fail_on_error=fail_on_error,
all_thumbnails=all_thumbnails,
)
manager.run()

Expand Down Expand Up @@ -233,6 +237,7 @@ def remotecontentimport(
exclude_node_ids=None,
update=False,
fail_on_error=False,
all_thumbnails=False,
):
manager_class = (
RemoteChannelUpdateManager if update else RemoteChannelResourceImportManager
Expand All @@ -244,6 +249,7 @@ def remotecontentimport(
node_ids=node_ids,
exclude_node_ids=exclude_node_ids,
fail_on_error=fail_on_error,
all_thumbnails=all_thumbnails,
)
manager.run()

Expand Down Expand Up @@ -434,6 +440,7 @@ def remoteimport(
exclude_node_ids=None,
update=False,
fail_on_error=False,
all_thumbnails=False,
):
call_command(
"importchannel",
Expand All @@ -457,6 +464,7 @@ def remoteimport(
node_ids=node_ids,
exclude_node_ids=exclude_node_ids,
fail_on_error=fail_on_error,
all_thumbnails=all_thumbnails,
)
manager.run()

Expand All @@ -471,7 +479,12 @@ def remoteimport(
status_fn=get_status,
)
def diskimport(
channel_id, drive_id, update=False, node_ids=None, exclude_node_ids=None
channel_id,
drive_id,
update=False,
node_ids=None,
exclude_node_ids=None,
all_thumbnails=False,
):
drive = get_mounted_drive_by_id(drive_id)
directory = drive.datafolder
Expand All @@ -498,6 +511,7 @@ def diskimport(
node_ids=node_ids,
exclude_node_ids=exclude_node_ids,
import_updates=update,
all_thumbnails=all_thumbnails,
)
manager.run()

Expand Down
4 changes: 2 additions & 2 deletions kolibri/core/content/test/test_annotation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -823,12 +823,12 @@ def test_set_all_files_all_exist(self, path_mock):

@patch(
"kolibri.core.content.utils.annotation.get_content_storage_file_path",
side_effect=[mock_content_file[1]] * 2 + [""] * 3,
side_effect=[mock_content_file[1]] * 2 + [""] * 8,
)
def test_set_all_files_two_exist(self, path_mock):
set_local_file_availability_from_disk()
self.assertEqual(LocalFile.objects.filter(available=True).count(), 2)
self.assertEqual(LocalFile.objects.exclude(available=True).count(), 3)
self.assertEqual(LocalFile.objects.exclude(available=True).count(), 8)

def test_set_bad_filenames(self):
local_files = list(LocalFile.objects.all())
Expand Down
2 changes: 1 addition & 1 deletion kolibri/core/content/test/test_content_app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1221,7 +1221,7 @@ def test_channelmetadata_has_exercises_filter(self):

def test_file_list(self):
response = self.client.get(reverse("kolibri:core:file-list"))
self.assertEqual(len(response.data), 5)
self.assertEqual(len(response.data), 10)

def test_file_retrieve(self):
response = self.client.get(
Expand Down
Loading