Remove need to iterate over file transfer objects as part of their ex…

…ternal API.

kolibri/core/content/management/commands/importchannel.py

@@ -198,8 +198,11 @@ def _start_file_transfer(
         with filetransfer, self.start_progress(
             total=filetransfer.transfer_size
         ) as progress_update:
-            for chunk in filetransfer:
-                progress_update(len(chunk), progress_extra_data)
+
+            def progress_callback(bytes):
+                progress_update(bytes, progress_extra_data)
+
+            filetransfer.run(progress_callback)
             # if upgrading, import the channel
             if not no_upgrade:
                 try:

kolibri/core/content/utils/resource_import.py

@@ -103,22 +103,14 @@ def _start_file_transfer(self, f, filetransfer):
         of data transferred. The error value will be None if no error
         occurred.
         """
-        data_transferred = 0
 
         with filetransfer:
             try:
-                for chunk in filetransfer:
-                    data_transferred += len(chunk)
+                filetransfer.run()
             except transfer.TransferFailed as e:
-                return e, data_transferred
-            # Ensure that if for some reason the total file size for the transfer
-            # is less than what we have marked in the database that we make up
-            # the difference so that the overall progress is never incorrect.
-            # This could happen, for example for a local transfer if a file
-            # has been replaced or corrupted (which we catch below)
-            data_transferred += f["file_size"] - filetransfer.transfer_size
+                return e, 0
 
-        return None, data_transferred
+        return None, f["file_size"] or 0
 
     @abstractmethod
     def get_import_data(self):
@@ -170,8 +162,8 @@ def _handle_future(self, future, f):
                 self.number_of_skipped_files += 1
             else:
                 self.file_checksums_to_annotate.append(f["id"])
-                self.transferred_file_size += f["file_size"]
-            self.remaining_bytes_to_transfer -= f["file_size"]
+                self.transferred_file_size += f["file_size"] or 0
+            self.remaining_bytes_to_transfer -= f["file_size"] or 0
             remaining_free_space = get_free_space(self.content_dir)
             if remaining_free_space <= self.remaining_bytes_to_transfer:
                 raise InsufficientStorageSpaceError(

kolibri/utils/file_transfer.py

@@ -368,6 +368,11 @@ def __iter__(self):
     def _get_content_iterator(self):
         pass
 
+    def run(self, progress_update=None):
+        for chunk in self:
+            if callable(progress_update):
+                progress_update(len(chunk))
+
     def _next(self):
         try:
             # Get the next chunk from the content iterator
@@ -534,7 +539,7 @@ def finalize_download(self):
         return (
             self._finalize_download
             and (self.range_start is None or self.range_start == 0)
-            and (self.range_end is None or self.range_end == self.file_size - 1)
+            and (self.range_end is None or self.range_end == self.total_size - 1)
         )
 
     def finalize(self):