Upload chunks in parallel

fixes: pulp#263
gerrod3 · Aug 18, 2021 · be5f008 · be5f008
1 parent 94a7c9c
commit be5f008
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/CHANGES/263.feature b/CHANGES/263.feature
@@ -0,0 +1 @@
+Artifact chunks are now uploaded in parallel
diff --git a/pulpcore/cli/core/context.py b/pulpcore/cli/core/context.py
@@ -74,20 +74,24 @@ def upload(self, file: IO[bytes], chunk_size: int = 1000000) -> Any:
         upload_href = upload_ctx.create(body={"size": size})["pulp_href"]
 
         try:
+            tasks = []
             while start < size:
                 end = min(size, start + chunk_size) - 1
                 file.seek(start)
                 chunk = file.read(chunk_size)
                 range_header = f"bytes {start}-{end}/{size}"
-                upload_ctx.update(
+                tasks.append(upload_ctx.update(
                     href=upload_href,
                     parameters={"Content-Range": range_header},
                     body={"sha256": hashlib.sha256(chunk).hexdigest()},
                     uploads={"file": chunk},
-                )
+                    non_blocking=True,
+                ))
                 start += chunk_size
-                click.echo(".", nl=False, err=True)
 
+            for task in tasks:
+                self.wait_for_task(task)
+                click.echo(".", nl=False, err=True)
             click.echo("Upload complete. Creating artifact.", err=True)
             task = upload_ctx.commit(
                 upload_href,