feat(blob): Provide onUploadProgress({ loaded, total, percentage })

.changeset/brown-years-heal.md

@@ -0,0 +1,25 @@
+---
+'@vercel/blob': minor
+'vercel-storage-integration-test-suite': minor
+---
+
+Add onUploadProgress feature to put/upload
+
+You can now track the upload progress in Node.js and all major browsers when
+using put/upload in multipart, non-multipart and client upload modes. Basically
+anywhere in our API you can upload a file, then you can follow the upload
+progress.
+
+Here's a basic usage example:
+
+```
+const blob = await put('big-file.pdf', file, {
+  access: 'public',
+  onUploadProgress(event) {
+    console.log(event.loaded, event.total, event.percentage);
+  }
+});
+```
+
+Fixes #543
+Fixes #642

packages/blob/jest/setup.js

@@ -2,5 +2,11 @@
 // but they are available everywhere else.
 // See https://stackoverflow.com/questions/68468203/why-am-i-getting-textencoder-is-not-defined-in-jest
 const { TextEncoder, TextDecoder } = require('node:util');
+// eslint-disable-next-line import/order -- On purpose to make requiring undici work
+const { ReadableStream } = require('node:stream/web');
 
-Object.assign(global, { TextDecoder, TextEncoder });
+Object.assign(global, { TextDecoder, TextEncoder, ReadableStream });
+
+const { Request, Response } = require('undici');
+
+Object.assign(global, { Request, Response });

packages/blob/package.json

@@ -63,22 +63,24 @@
     "async-retry": "^1.3.3",
     "bytes": "^3.1.2",
     "is-buffer": "^2.0.5",
+    "is-node-process": "^1.2.0",
+    "throttleit": "^2.1.0",
     "undici": "^5.28.4"
   },
   "devDependencies": {
     "@edge-runtime/jest-environment": "2.3.10",
     "@edge-runtime/types": "2.2.9",
-    "@types/async-retry": "1.4.8",
+    "@types/async-retry": "1.4.9",
     "@types/bytes": "3.1.4",
-    "@types/jest": "29.5.13",
-    "@types/node": "22.7.3",
+    "@types/jest": "29.5.14",
+    "@types/node": "22.9.0",
     "eslint": "8.56.0",
     "eslint-config-custom": "workspace:*",
     "jest": "29.7.0",
     "jest-environment-jsdom": "29.7.0",
     "ts-jest": "29.2.5",
     "tsconfig": "workspace:*",
-    "tsup": "8.3.0"
+    "tsup": "8.3.5"
   },
   "engines": {
     "node": ">=16.14"

packages/blob/src/api.ts

@@ -1,9 +1,20 @@
-import type { RequestInit, Response } from 'undici';
-import { fetch } from 'undici';
+import type { Response } from 'undici';
 import retry from 'async-retry';
+import isNetworkError from './is-network-error';
 import { debug } from './debug';
-import type { BlobCommandOptions } from './helpers';
-import { BlobError, getTokenFromOptionsOrEnv } from './helpers';
+import type {
+  BlobCommandOptions,
+  BlobRequestInit,
+  WithUploadProgress,
+} from './helpers';
+import {
+  BlobError,
+  computeBodyLength,
+  getApiUrl,
+  getTokenFromOptionsOrEnv,
+} from './helpers';
+import { blobRequest } from './request';
+import { DOMException } from './dom-exception';
 
 // maximum pathname length is:
 // 1024 (provider limit) - 26 chars (vercel  internal suffixes) - 31 chars (blob `-randomId` suffix) = 967
@@ -132,20 +143,6 @@ function getApiVersion(): string {
   return `${versionOverride ?? BLOB_API_VERSION}`;
 }
 
-function getApiUrl(pathname = ''): string {
-  let baseUrl = null;
-  try {
-    // wrapping this code in a try/catch as this function is used in the browser and Vite doesn't define the process.env.
-    // As this varaible is NOT used in production, it will always default to production endpoint
-    baseUrl =
-      process.env.VERCEL_BLOB_API_URL ||
-      process.env.NEXT_PUBLIC_VERCEL_BLOB_API_URL;
-  } catch {
-    // noop
-  }
-  return `${baseUrl || 'https://blob.vercel-storage.com'}${pathname}`;
-}
-
 function getRetries(): number {
   try {
     const retries = process.env.VERCEL_BLOB_RETRIES || '10';
@@ -175,7 +172,6 @@ async function getBlobError(
 
   try {
     const data = (await response.json()) as BlobApiError;
-
     code = data.error?.code ?? 'unknown_error';
     message = data.error?.message;
   } catch {
@@ -254,8 +250,8 @@ async function getBlobError(
 
 export async function requestApi<TResponse>(
   pathname: string,
-  init: RequestInit,
-  commandOptions: BlobCommandOptions | undefined,
+  init: BlobRequestInit,
+  commandOptions: (BlobCommandOptions & WithUploadProgress) | undefined,
 ): Promise<TResponse> {
   const apiVersion = getApiVersion();
   const token = getTokenFromOptionsOrEnv(commandOptions);
@@ -264,23 +260,75 @@ export async function requestApi<TResponse>(
   const [, , , storeId = ''] = token.split('_');
   const requestId = `${storeId}:${Date.now()}:${Math.random().toString(16).slice(2)}`;
   let retryCount = 0;
+  let bodyLength = 0;
+  let totalLoaded = 0;
+  const sendBodyLength =
+    commandOptions?.onUploadProgress || shouldUseXContentLength();
+
+  if (
+    init.body &&
+    // 1. For upload progress we always need to know the total size of the body
+    // 2. In development we need the header for put() to work correctly when passing a stream
+    sendBodyLength
+  ) {
+    bodyLength = computeBodyLength(init.body);
+  }
+
+  if (commandOptions?.onUploadProgress) {
+    commandOptions.onUploadProgress({
+      loaded: 0,
+      total: bodyLength,
+      percentage: 0,
+    });
+  }
 
   const apiResponse = await retry(
     async (bail) => {
       let res: Response;
 
       // try/catch here to treat certain errors as not-retryable
       try {
-        res = await fetch(getApiUrl(pathname), {
-          ...init,
-          headers: {
-            'x-api-blob-request-id': requestId,
-            'x-api-blob-request-attempt': String(retryCount),
-            'x-api-version': apiVersion,
-            authorization: `Bearer ${token}`,
-            ...extraHeaders,
-            ...init.headers,
+        res = await blobRequest({
+          input: getApiUrl(pathname),
+          init: {
+            ...init,
+            headers: {
+              'x-api-blob-request-id': requestId,
+              'x-api-blob-request-attempt': String(retryCount),
+              'x-api-version': apiVersion,
+              ...(sendBodyLength
+                ? { 'x-content-length': String(bodyLength) }
+                : {}),
+              authorization: `Bearer ${token}`,
+              ...extraHeaders,
+              ...init.headers,
+            },
           },
+          onUploadProgress: commandOptions?.onUploadProgress
+            ? (loaded) => {
+                const total = bodyLength !== 0 ? bodyLength : loaded;
+                totalLoaded = loaded;
+                const percentage =
+                  bodyLength > 0
+                    ? Number(((loaded / total) * 100).toFixed(2))
+                    : 0;
+
+                // Leave percentage 100 for the end of request
+                if (percentage === 100 && bodyLength > 0) {
+                  return;
+                }
+
+                commandOptions.onUploadProgress?.({
+                  loaded,
+                  // When passing a stream to put(), we have no way to know the total size of the body.
+                  // Instead of defining total as total?: number we decided to set the total to the currently
+                  // loaded number. This is not inaccurate and way more practical for DX.
+                  // Passing down a stream to put() is very rare
+                  total,
+                  percentage,
+                });
+              }
+            : undefined,
         });
       } catch (error) {
         // if the request was aborted, don't retry
@@ -289,6 +337,18 @@ export async function requestApi<TResponse>(
           return;
         }
 
+        // We specifically target network errors because fetch network errors are regular TypeErrors
+        // We want to retry for network errors, but not for other TypeErrors
+        if (isNetworkError(error)) {
+          throw error;
+        }
+
+        // If we messed up the request part, don't even retry
+        if (error instanceof TypeError) {
+          bail(error);
+          return;
+        }
+
         // retry for any other erros thrown by fetch
         throw error;
       }
@@ -314,7 +374,10 @@ export async function requestApi<TResponse>(
     {
       retries: getRetries(),
       onRetry: (error) => {
-        debug(`retrying API request to ${pathname}`, error.message);
+        if (error instanceof Error) {
+          debug(`retrying API request to ${pathname}`, error.message);
+        }
+
         retryCount = retryCount + 1;
       },
     },
@@ -324,6 +387,20 @@ export async function requestApi<TResponse>(
     throw new BlobUnknownError();
   }
 
+  // Calling onUploadProgress here has two benefits:
+  // 1. It ensures 100% is only reached at the end of the request. While otherwise you can reach 100%
+  // before the request is fully done, as we only really measure what gets sent over the wire, not what
+  // has been processed by the server.
+  // 2. It makes the uploadProgress "work" even in rare cases where fetch/xhr onprogress is not working
+  // And in the case of multipart uploads it actually provides a simple progress indication (per part)
+  if (commandOptions?.onUploadProgress) {
+    commandOptions.onUploadProgress({
+      loaded: totalLoaded,
+      total: totalLoaded,
+      percentage: 100,
+    });
+  }
+
   return (await apiResponse.json()) as TResponse;
 }
 
@@ -333,20 +410,31 @@ function getProxyThroughAlternativeApiHeaderFromEnv(): {
   const extraHeaders: Record<string, string> = {};
 
   try {
-    if ('VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API' in process.env) {
+    if (
+      'VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API' in process.env &&
+      process.env.VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API !== undefined
+    ) {
       extraHeaders['x-proxy-through-alternative-api'] =
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- we know it's here from the if
-        process.env.VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API!;
+        process.env.VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API;
     } else if (
-      'NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API' in process.env
+      'NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API' in process.env &&
+      process.env.NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API !==
+        undefined
     ) {
       extraHeaders['x-proxy-through-alternative-api'] =
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- we know it's here from the if
-        process.env.NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API!;
+        process.env.NEXT_PUBLIC_VERCEL_BLOB_PROXY_THROUGH_ALTERNATIVE_API;
     }
   } catch {
     // noop
   }
 
   return extraHeaders;
 }
+
+function shouldUseXContentLength(): boolean {
+  try {
+    return process.env.VERCEL_BLOB_USE_X_CONTENT_LENGTH === '1';
+  } catch {
+    return false;
+  }
+}

packages/blob/src/client.browser.test.ts

@@ -88,7 +88,6 @@ describe('client', () => {
         'https://blob.vercel-storage.com/foo.txt',
         {
           body: 'Test file data',
-          duplex: 'half',
           headers: {
             authorization: 'Bearer vercel_blob_client_fake_123',
             'x-api-blob-request-attempt': '0',
@@ -232,7 +231,6 @@ describe('client', () => {
             'x-mpu-part-number': '1',
           },
           method: 'POST',
-          duplex: 'half',
           signal: internalAbortSignal,
         },
       );
@@ -252,7 +250,6 @@ describe('client', () => {
             'x-mpu-part-number': '2',
           },
           method: 'POST',
-          duplex: 'half',
           signal: internalAbortSignal,
         },
       );
@@ -376,7 +373,6 @@ describe('client', () => {
             'x-mpu-part-number': '1',
           },
           method: 'POST',
-          duplex: 'half',
           signal: internalAbortSignal,
         },
       );
@@ -396,7 +392,6 @@ describe('client', () => {
             'x-mpu-part-number': '2',
           },
           method: 'POST',
-          duplex: 'half',
           signal: internalAbortSignal,
         },
       );

packages/blob/src/client.ts

@@ -5,7 +5,7 @@ import type { IncomingMessage } from 'node:http';
 // the `undici` module will be replaced with https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API
 // for browser contexts. See ./undici-browser.js and ./package.json
 import { fetch } from 'undici';
-import type { BlobCommandOptions } from './helpers';
+import type { BlobCommandOptions, WithUploadProgress } from './helpers';
 import { BlobError, getTokenFromOptionsOrEnv } from './helpers';
 import { createPutMethod } from './put';
 import type { PutBlobResult } from './put-helpers';
@@ -42,7 +42,9 @@ export interface ClientTokenOptions {
 }
 
 // shared interface for put and upload
-interface ClientCommonPutOptions extends ClientCommonCreateBlobOptions {
+interface ClientCommonPutOptions
+  extends ClientCommonCreateBlobOptions,
+    WithUploadProgress {
   /**
    * Whether to use multipart upload. Use this when uploading large files. It will split the file into multiple parts, upload them in parallel and retry failed parts.
    */
@@ -89,7 +91,7 @@ export const put = createPutMethod<ClientPutCommandOptions>({
 // vercelBlob. createMultipartUpload()
 // vercelBlob. uploadPart()
 // vercelBlob. completeMultipartUpload()
-// vercelBlob. createMultipartUploaded()
+// vercelBlob. createMultipartUploader()
 
 export type ClientCreateMultipartUploadCommandOptions =
   ClientCommonCreateBlobOptions & ClientTokenOptions;
@@ -110,7 +112,8 @@ export const createMultipartUploader =
 
 type ClientMultipartUploadCommandOptions = ClientCommonCreateBlobOptions &
   ClientTokenOptions &
-  CommonMultipartUploadOptions;
+  CommonMultipartUploadOptions &
+  WithUploadProgress;
 
 export const uploadPart =
   createUploadPartMethod<ClientMultipartUploadCommandOptions>({