Set the admin imported flag during resource import.

kolibri/core/content/test/test_annotation.py

@@ -375,7 +375,7 @@ def test_all_local_files_available_exclude_duplicate_topic(self):
             ContentNode.objects.filter(title="c2c1", available=True).count(), 1
         )
 
-    def test_all_local_files_available_include_orignial_exclude_duplicate_topic(self):
+    def test_all_local_files_available_include_original_exclude_duplicate_topic(self):
         ContentNode.objects.all().update(available=False)
         LocalFile.objects.all().update(available=True)
         parent = ContentNode.objects.get(title="c3")
@@ -419,6 +419,76 @@ def test_all_local_files_available_non_include_exclude_unaffected(self):
         node.refresh_from_db()
         self.assertTrue(node.available)
 
+    def test_all_local_files_admin_imported_true(self):
+        ContentNode.objects.all().update(available=False)
+        LocalFile.objects.all().update(available=True)
+        node = ContentNode.objects.get(title="copy", kind=content_kinds.VIDEO)
+        set_leaf_node_availability_from_local_file_availability(
+            test_channel_id, admin_imported=True
+        )
+        node.refresh_from_db()
+        self.assertTrue(node.admin_imported)
+
+    def test_all_local_files_admin_imported_true_overwrites(self):
+        ContentNode.objects.all().update(available=False)
+        LocalFile.objects.all().update(available=True)
+        node = ContentNode.objects.get(title="copy", kind=content_kinds.VIDEO)
+        node.admin_imported = False
+        node.save()
+        set_leaf_node_availability_from_local_file_availability(
+            test_channel_id, admin_imported=True
+        )
+        node.refresh_from_db()
+        self.assertTrue(node.admin_imported)
+
+    def test_all_local_files_admin_imported_false(self):
+        ContentNode.objects.all().update(available=False)
+        LocalFile.objects.all().update(available=True)
+        node = ContentNode.objects.get(title="copy", kind=content_kinds.VIDEO)
+        set_leaf_node_availability_from_local_file_availability(
+            test_channel_id, admin_imported=False
+        )
+        node.refresh_from_db()
+        self.assertFalse(node.admin_imported)
+
+    def test_all_local_files_admin_imported_false_no_overwrite(self):
+        ContentNode.objects.all().update(available=False)
+        LocalFile.objects.all().update(available=True)
+        node = ContentNode.objects.get(title="copy", kind=content_kinds.VIDEO)
+        node.admin_imported = True
+        node.save()
+        set_leaf_node_availability_from_local_file_availability(
+            test_channel_id, admin_imported=False
+        )
+        node.refresh_from_db()
+        self.assertTrue(node.admin_imported)
+
+    def test_all_local_files_admin_imported_none_no_effect(self):
+        ContentNode.objects.all().update(available=False)
+        LocalFile.objects.all().update(available=True)
+        node = ContentNode.objects.get(title="copy", kind=content_kinds.VIDEO)
+        set_leaf_node_availability_from_local_file_availability(test_channel_id)
+        node.refresh_from_db()
+        self.assertIsNone(node.admin_imported)
+
+    def test_all_local_files_admin_imported_non_include_exclude_unaffected(self):
+        ContentNode.objects.all().update(available=False)
+        LocalFile.objects.all().update(available=True)
+        exclude = ContentNode.objects.get(title="c3")
+        include = ContentNode.objects.get(title="c2")
+        node = ContentNode.objects.get(title="copy", kind=content_kinds.VIDEO)
+        self.assertFalse(node.available)
+        node.admin_imported = False
+        node.save()
+        set_leaf_node_availability_from_local_file_availability(
+            test_channel_id,
+            node_ids=[include.id],
+            exclude_node_ids=[exclude.id],
+            admin_imported=True,
+        )
+        node.refresh_from_db()
+        self.assertFalse(node.admin_imported)
+
     def tearDown(self):
         call_command("flush", interactive=False)
         super(AnnotationFromLocalFileAvailability, self).tearDown()

kolibri/core/content/test/test_import_export.py

@@ -1007,6 +1007,7 @@ def test_remote_import_httperror_404(
             node_ids={self.c2c1_node_id},
             exclude_node_ids=None,
             public=False,
+            admin_imported=True,
         )
 
     @patch("kolibri.core.content.utils.resource_import.transfer.FileDownload")
@@ -1032,7 +1033,12 @@ def test_remote_import_httperror_500(
             manager = RemoteChannelResourceImportManager(self.the_channel_id)
             manager.run()
         self.annotation_mock.set_content_visibility.assert_called_with(
-            self.the_channel_id, [], node_ids=None, exclude_node_ids=None, public=False
+            self.the_channel_id,
+            [],
+            node_ids=None,
+            exclude_node_ids=None,
+            public=False,
+            admin_imported=True,
         )
 
     @patch("kolibri.core.content.utils.resource_import.get_free_space")
@@ -1140,6 +1146,7 @@ def test_remote_import_no_space_after_first_download(
             exclude_node_ids=None,
             node_ids=None,
             public=False,
+            admin_imported=True,
         )
 
     @patch("kolibri.utils.file_transfer.sleep")
@@ -1422,6 +1429,7 @@ def test_remote_import_source_corrupted(
             exclude_node_ids=None,
             node_ids={self.c1_node_id},
             public=False,
+            admin_imported=True,
         )
 
     @patch(
@@ -1480,6 +1488,7 @@ def test_remote_import_full_import(
             exclude_node_ids=None,
             node_ids=None,
             public=False,
+            admin_imported=True,
         )
 
     def test_local_import_with_detected_manifest_file(
@@ -1988,6 +1997,7 @@ def test_remote_import_fail_on_error_missing(
             node_ids={self.c2c1_node_id},
             exclude_node_ids=None,
             public=False,
+            admin_imported=True,
         )
 
     @patch("kolibri.core.content.utils.resource_import.logger.warning")

kolibri/core/content/utils/annotation.py

@@ -18,6 +18,7 @@
 from sqlalchemy import select
 from sqlalchemy import String
 from sqlalchemy.sql.expression import literal
+from sqlalchemy.sql.functions import coalesce
 
 from .paths import get_content_file_name
 from .paths import get_content_storage_file_path
@@ -300,7 +301,7 @@ def set_leaf_nodes_invisible(channel_id, node_ids=None, exclude_node_ids=None):
 
 
 def set_leaf_node_availability_from_local_file_availability(
-    channel_id, node_ids=None, exclude_node_ids=None
+    channel_id, node_ids=None, exclude_node_ids=None, admin_imported=None
 ):
     """
     Set nodes in a channel as available, based on their required files.
@@ -369,6 +370,25 @@ def set_leaf_node_availability_from_local_file_availability(
         )
     )
 
+    values_dict = {
+        "available": exists(contentnode_statement),
+    }
+
+    if admin_imported is not None:
+        # Expression that will resolve a boolean value for admin_imported
+        # Everything after the select statement should be identical to the available_nodes expression above.
+        if bridge.engine.name == "sqlite":
+            # Use a max function to simulate an OR.
+            admin_imported_statement = func.max(
+                admin_imported, coalesce(ContentNodeTable.c.admin_imported, False)
+            )
+        elif bridge.engine.name == "postgresql":
+            # Use the postgres boolean OR operator
+            admin_imported_statement = func.bool_or(
+                admin_imported, coalesce(ContentNodeTable.c.admin_imported, False)
+            )
+        values_dict["admin_imported"] = admin_imported_statement
+
     while min_boundary < max_rght:
         batch_statement = _create_batch_update_statement(
             bridge,
@@ -381,9 +401,7 @@ def set_leaf_node_availability_from_local_file_availability(
 
         # Execute the update for this batch
         connection.execute(
-            batch_statement.values(
-                available=exists(contentnode_statement)
-            ).execution_options(autocommit=True)
+            batch_statement.values(**values_dict).execution_options(autocommit=True)
         )
         min_boundary += dynamic_chunksize
 
@@ -691,10 +709,13 @@ def reannotate_all_channels():
 
 
 def update_content_metadata(
-    channel_id, node_ids=None, exclude_node_ids=None, public=None
+    channel_id, node_ids=None, exclude_node_ids=None, public=None, admin_imported=None
 ):
     set_leaf_node_availability_from_local_file_availability(
-        channel_id, node_ids=node_ids, exclude_node_ids=exclude_node_ids
+        channel_id,
+        node_ids=node_ids,
+        exclude_node_ids=exclude_node_ids,
+        admin_imported=admin_imported,
     )
     recurse_annotation_up_tree(channel_id)
     set_channel_metadata_fields(channel_id, public=public)
@@ -705,11 +726,20 @@ def update_content_metadata(
 
 
 def set_content_visibility(
-    channel_id, checksums, node_ids=None, exclude_node_ids=None, public=None
+    channel_id,
+    checksums,
+    node_ids=None,
+    exclude_node_ids=None,
+    public=None,
+    admin_imported=None,
 ):
     mark_local_files_as_available(checksums)
     update_content_metadata(
-        channel_id, node_ids=node_ids, exclude_node_ids=exclude_node_ids, public=public
+        channel_id,
+        node_ids=node_ids,
+        exclude_node_ids=exclude_node_ids,
+        public=public,
+        admin_imported=admin_imported,
     )
 
 

kolibri/core/content/utils/resource_import.py

@@ -66,6 +66,7 @@ def __init__(
         all_thumbnails=False,
         fail_on_error=False,
         content_dir=None,
+        admin_imported=True,
     ):
         self.channel_id = channel_id
 
@@ -81,6 +82,7 @@ def __init__(
         self.all_thumbnails = all_thumbnails
         self.fail_on_error = fail_on_error
         self.content_dir = content_dir or conf.OPTIONS["Paths"]["CONTENT_DIR"]
+        self.admin_imported = admin_imported
         super(ResourceImportManagerBase, self).__init__()
 
     @classmethod
@@ -301,6 +303,7 @@ def run_import(self):
             node_ids=self.node_ids,
             exclude_node_ids=self.exclude_node_ids,
             public=self.public,
+            admin_imported=self.admin_imported,
         )
 
         self.resources_after_transfer = (
@@ -347,6 +350,7 @@ def __init__(
         all_thumbnails=False,
         fail_on_error=False,
         content_dir=None,
+        admin_imported=True,
         timeout=transfer.Transfer.DEFAULT_TIMEOUT,
     ):
         super(RemoteResourceImportManagerBase, self).__init__(
@@ -357,6 +361,7 @@ def __init__(
             all_thumbnails=all_thumbnails,
             fail_on_error=fail_on_error,
             content_dir=content_dir,
+            admin_imported=admin_imported,
         )
         self.timeout = timeout
         self.peer_id = peer_id
@@ -405,6 +410,7 @@ def __init__(
         all_thumbnails=False,
         fail_on_error=False,
         content_dir=None,
+        admin_imported=True,
     ):
         self.drive_id = drive_id
         if drive_id and not path:
@@ -420,6 +426,7 @@ def __init__(
             all_thumbnails=all_thumbnails,
             fail_on_error=fail_on_error,
             content_dir=content_dir,
+            admin_imported=admin_imported,
         )
 
     @staticmethod
@@ -523,6 +530,9 @@ def __init__(
         fail_on_error=False,
         content_dir=None,
         timeout=transfer.Transfer.DEFAULT_TIMEOUT,
+        # As this is primarily used for importing non-admin imported content
+        # we reverse the default here.
+        admin_imported=False,
     ):
         """
         :param channel_id: A hex UUID string
@@ -549,6 +559,7 @@ def __init__(
             renderable_only=renderable_only,
             fail_on_error=fail_on_error,
             content_dir=content_dir,
+            admin_imported=admin_imported,
             timeout=timeout,
         )
         self.peer = peer