fix: improve batch upload progress data (#566)

* chore: increase core bundle allowed size
rpldy · Sep 10, 2023 · 6793f02 · 6793f02
1 parent ed2a555
commit 6793f02
Show file tree

Hide file tree

Showing 11 changed files with 603 additions and 865 deletions.
diff --git a/.storybook/main.js b/.storybook/main.js
@@ -19,7 +19,16 @@ module.exports = {
 
     addons: [
         path.resolve("./.storybook/uploadyPreset"),
-        getAbsolutePath("@storybook/addon-essentials"),
+        // getAbsolutePath("@storybook/addon-essentials"),
+        {
+            name: "@storybook/addon-essentials",
+            options: {
+                docs: true,
+                actions: true,
+                backgrounds: true,
+                controls: true
+            }
+        },
         getAbsolutePath("@storybook/addon-knobs"),
     ],
 

diff --git a/bundle.config.js b/bundle.config.js
@@ -40,7 +40,7 @@ module.exports = {
             "core": {
                 pkgs: [PKGS.LIFE_EVENTS, PKGS.SHARED, PKGS.SENDER, PKGS.UPLOADER],
                 target: PKGS.UPLOADER,
-                maxSize: 11500,
+                maxSize: 12000,
                 dontUsePolyfills: true,
             },
 

diff --git a/packages/core/shared/src/types.js b/packages/core/shared/src/types.js
@@ -83,10 +83,12 @@ export type Batch = {
 	uploaderId: string,
 	items: BatchItem[],
 	state: BatchState,
-    //average of percentage of upload completed for batch items
+    //percentage of upload completed for batch items
     completed: number,
     //sum of bytes uploaded for batch items
     loaded: number,
+    //sum of bytes of all items in the batch
+    total: number,
     //number of items originally added to batch when its created
     orgItemCount: number,
     additionalInfo: ?string,

diff --git a/packages/core/shared/types/index.d.ts b/packages/core/shared/types/index.d.ts
@@ -59,6 +59,7 @@ export type Batch = {
     state: BATCH_STATES;
     completed: number;
     loaded: number;
+    total: number;
     orgItemCount: number;
     additionalInfo: string | null;
 };

diff --git a/packages/core/uploader/src/batch.js b/packages/core/uploader/src/batch.js
@@ -50,6 +50,7 @@ const createBatch = (files: UploadInfo | UploadInfo[], uploaderId: string, optio
     const isPending = !options.autoUpload;
 
     return processFiles(id, usedFiles, isPending, options.fileFilter)
+        // $FlowIgnore[incompatible-call]  flow went crazy here :(
         .then((items) => {
             return {
                 id,

diff --git a/packages/core/uploader/src/batchItemsSender.js b/packages/core/uploader/src/batchItemsSender.js
@@ -24,7 +24,9 @@ const reportItemsProgress = (items: BatchItem[], completed: number, loaded: numb
 
 const onItemUploadProgress = (items: BatchItem[], batch: Batch, e: ProgressEvent, trigger: TriggerMethod) => {
     const completed = Math.min(((e.loaded / e.total) * 100), 100),
+        //average the bytes loaded across the files in the group
         completedPerItem = completed / items.length,
+        //average the percentage across the files in the group
         loadedAverage = e.loaded / items.length;
 
     reportItemsProgress(items, completedPerItem, loadedAverage, e.total, trigger);

diff --git a/packages/core/uploader/src/queue/tests/uploaderQueue.test.js b/packages/core/uploader/src/queue/tests/uploaderQueue.test.js
@@ -284,8 +284,10 @@ describe("queue tests", () => {
             const batch = {
                     id: "b1",
                     items: [
-                        { id: "u1", completed: 20, loaded: 1000 },
-                        { id: "u2", completed: 80, loaded: 3000 }
+                        { id: "u1", completed: 20, loaded: 1000, file: { size: 5000 } },
+                        { id: "u2", completed: 75, loaded: 3200, file: { size: 4000 }  },
+                        { id: "u3", completed: 40, loaded: 8000, file: { size: 20000 } },
+                        { id: "u4", completed: 100, loaded: 1, file: undefined, url: "test" },
                     ]
                 },
                 batchOptions = {};
@@ -296,8 +298,10 @@ describe("queue tests", () => {
             senderOnHandlers[SENDER_EVENTS.BATCH_PROGRESS](batch);
 
             const state2 = queue.getState();
-            expect(state2.batches["b1"].batch.completed).toBe(50);
-            expect(state2.batches["b1"].batch.loaded).toBe(4000);
+
+            expect(state2.batches["b1"].batch.loaded).toBe(12201);
+            expect(state2.batches["b1"].batch.total).toBe(29001);
+            expect( state2.batches["b1"].batch.completed).toBeCloseTo(0.42, 2);
 
             expect(triggerUploaderBatchEvent).toHaveBeenCalledWith(expect.any(Object), "b1", UPLOADER_EVENTS.BATCH_PROGRESS);
         });

diff --git a/packages/core/uploader/src/queue/uploaderQueue.js b/packages/core/uploader/src/queue/uploaderQueue.js
@@ -104,21 +104,26 @@ const createUploaderQueue = (
             .items;
 
         if (batchItems) {
-            const [completed, loaded] = batchItems
+            const [loaded, total] = batchItems
                 .reduce((res, { id }) => {
                     //getting data from state.items since in dev the wrapped state.batch.items and state.items aren't the same objects
-                    const { completed, loaded } = state.items[id];
-                    res[0] += completed;
-                    res[1] += loaded;
+                    const { loaded, file } = state.items[id];
+                    const size = file?.size || loaded || 1;
+                    //loaded = uploaded bytes
+                    res[0] += loaded;
+                    //total = file byte size
+                    res[1] += size;
                     return res;
                 }, [0, 0]);
 
             updateState((state: State) => {
                 const stateBatch = state.batches[batch.id].batch;
-                //average of completed percentage for batch items
-                stateBatch.completed = completed / batchItems.length;
+                //sum of bytes in batch items
+                stateBatch.total = total;
                 //sum of loaded bytes for batch items
                 stateBatch.loaded = loaded;
+                //completed percentage for batch items
+                stateBatch.completed = loaded / total;
             });
 
             triggerUploaderBatchEvent(queueState, batch.id, UPLOADER_EVENTS.BATCH_PROGRESS);

diff --git a/packages/ui/uploady/Uploady.stories.js b/packages/ui/uploady/Uploady.stories.js
@@ -25,6 +25,7 @@ import Uploady, {
     useUploadOptions,
     useBatchStartListener,
     useBatchAddListener,
+    useBatchProgressListener,
     useItemStartListener,
     useItemFinishListener,
     useItemErrorListener,
@@ -46,6 +47,10 @@ const ContextUploadButton = ({ text = "Custom Upload Button" }: { text?: string,
         uploadyContext?.showFileUpload();
     }, [uploadyContext]);
 
+    // useBatchProgressListener(({ completed, loaded, total}) => {
+    //     console.log("------- BATCH PROGRESS --------------", { completed, loaded, total })
+    // });
+
     return <button id="upload-button" onClick={onClick}>{text}</button>;
 };
 

diff --git a/server/package.json b/server/package.json
@@ -4,9 +4,9 @@
     "version": "1.0.0",
     "devDependencies": {
         "cors": "^2.8.5",
-        "express": "^4.17.3",
-        "express-fileupload": "^1.1.6",
+        "express": "^4.18.2",
+        "express-fileupload": "^1.4.0",
         "lodash": "^4.17.21",
-        "tus-node-server": "^0.3.2"
+        "tus-node-server": "^0.9.0"
     }
 }