feat: upgrade s3 to v3 (#31)

planjs · Dec 8, 2023 · e557def · e557def
1 parent 1e72c5d
commit e557def
Show file tree

Hide file tree

Showing 6 changed files with 1,148 additions and 210 deletions.
diff --git a/packages/oss-upload-tool/package.json b/packages/oss-upload-tool/package.json
@@ -39,7 +39,8 @@
     "@planjs/utils": "^1.5.1",
     "@types/ali-oss": "^6.16.3",
     "ali-oss": "^6.17.1",
-    "aws-sdk": "^2.1294.0",
+    "@aws-sdk/client-s3": "3.449.0",
+    "@aws-sdk/lib-storage": "3.449.0",
     "cos-nodejs-sdk-v5": "^2.11.9",
     "stan-utils": "0.15.0"
   },

diff --git a/packages/oss-upload-tool/src/client/ali.ts b/packages/oss-upload-tool/src/client/ali.ts
@@ -1,4 +1,3 @@
-import { join } from 'path';
 import { URL } from 'url';
 
 import AOSS from 'ali-oss';

diff --git a/packages/oss-upload-tool/src/client/cos.ts b/packages/oss-upload-tool/src/client/cos.ts
@@ -1,5 +1,3 @@
-import { join } from 'path';
-
 import COS from 'cos-nodejs-sdk-v5';
 import { REG_URI } from '@planjs/utils';
 import { lodash } from 'stan-utils';

diff --git a/packages/oss-upload-tool/src/client/s3.ts b/packages/oss-upload-tool/src/client/s3.ts
@@ -1,8 +1,10 @@
 import { createReadStream } from 'fs';
 import { URL } from 'url';
 
-import { S3 } from 'aws-sdk';
-import { defer } from '@planjs/utils';
+import { S3Client as S3 } from '@aws-sdk/client-s3';
+import type { S3ClientConfig, PutObjectCommandInput } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
+import { AbortController } from '@smithy/abort-controller';
 import { lodash } from 'stan-utils';
 
 import { Client } from '../oss_client';
@@ -22,7 +24,7 @@ import {
   UPLOAD_TIMEOUT_KEY,
 } from '../consts';
 
-class S3Client extends Client<Partial<S3.Types.ClientConfiguration>, S3.Types.PutObjectRequest> {
+class S3Client extends Client<Partial<S3ClientConfig>, PutObjectCommandInput> {
   readonly #client!: S3;
 
   constructor(options: OSSUploadOptions) {
@@ -46,66 +48,73 @@ class S3Client extends Client<Partial<S3.Types.ClientConfiguration>, S3.Types.Pu
     };
   };
 
-  get globalOptions(): Partial<Partial<S3.ClientConfiguration>> {
+  get globalOptions(): Partial<Partial<S3ClientConfig>> {
     const accessKeyId = getGlobalValue(S3_SECRET_ID, SECRET_ID)!;
     const secretAccessKey = getGlobalValue(S3_SECRET_KEY, SECRET_KEY)!;
     const region = getGlobalValue(S3_REGION_KEY, BUCKET_KEY)!;
-    const timeout = getGlobalValue(UPLOAD_TIMEOUT_KEY);
     return {
       credentials: {
         accessKeyId,
         secretAccessKey,
       },
       region,
-      httpOptions: {
-        timeout: Number.isNaN(+timeout!)
-          ? defaultVal(this.opt.timeout, DEFAULT_TIMEOUT)
-          : +timeout!,
-      },
     };
   }
 
-  get globalUploadParams(): Partial<S3.Types.PutObjectRequest> {
+  get globalUploadParams(): Partial<PutObjectCommandInput> {
     const Bucket = getGlobalValue(S3_BUCKET_KEY, REGION_KEY)!;
     return {
       Bucket,
     };
   }
 
-  upload = (
+  getTimeout() {
+    const timeout = getGlobalValue(UPLOAD_TIMEOUT_KEY);
+
+    return Number.isNaN(+timeout!) ? defaultVal(this.opt.timeout, DEFAULT_TIMEOUT) : +timeout!;
+  }
+
+  upload = async (
     item: OSSUploadLocalItem,
-    params?: Partial<S3.Types.PutObjectRequest>,
+    params?: Partial<PutObjectCommandInput>,
     options?: UploadOptions,
   ): Promise<UploadResp> => {
-    const p = defer<UploadResp>();
-    this.#client
-      .upload(
-        {
-          Bucket: '',
-          Key: item.path,
-          Body: createReadStream(item.filePath),
-          ...this.globalUploadParams,
-          ...params,
-        },
-        {
-          partSize: 1024 * 1024 * 5,
-          ...options,
-        },
-        (err, data) => {
-          if (err) {
-            p.reject(err);
-            return;
-          }
-          p.resolve({
-            ...data,
-            url: data.Location,
-          });
-        },
-      )
-      .on('httpUploadProgress', (info) => {
-        options?.onProgress?.(info.loaded, info.total);
-      });
-    return p.promise;
+    const _params = {
+      Bucket: '',
+      Key: item.path,
+      Body: createReadStream(item.filePath),
+      ...this.globalUploadParams,
+      ...params,
+    };
+
+    const abortController = new AbortController();
+
+    const parallelUploads3 = new Upload({
+      client: this.#client,
+      params: _params,
+      queueSize: 4,
+      partSize: 1024 * 1024 * 5,
+      leavePartsOnError: false,
+      abortController,
+    });
+
+    const timer = setTimeout(() => {
+      abortController.abort();
+    }, this.getTimeout());
+
+    parallelUploads3.on('httpUploadProgress', (progress) => {
+      options?.onProgress?.(progress.loaded! || 0, progress.total! || 0);
+    });
+
+    try {
+      const res = await parallelUploads3.done();
+      return {
+        ...res,
+        ...(await this.getUploadedUrl(item, _params)),
+      } as UploadResp;
+    } finally {
+      clearTimeout(timer);
+    }
   };
 }
 

diff --git a/packages/oss-upload-tool/src/types.ts b/packages/oss-upload-tool/src/types.ts
@@ -1,14 +1,14 @@
 import type COS from 'cos-nodejs-sdk-v5';
 import type AOSS from 'ali-oss';
-import type { S3 } from 'aws-sdk';
+import type { S3ClientConfig, PutObjectCommandInput } from '@aws-sdk/client-s3';
 
 /**
  * 上传参数
  */
 type UploadParams =
   | Partial<COS.UploadFileParams>
   | Partial<AOSS.PutObjectOptions>
-  | Partial<S3.Types.PutObjectRequest>;
+  | Partial<PutObjectCommandInput>;
 
 /**
  * 对象存储类型
@@ -73,7 +73,7 @@ export interface OSSUploadOptions {
   /**
    * S3 初始化参数
    */
-  S3Options?: S3.Types.ClientConfiguration;
+  S3Options?: S3ClientConfig;
   /**
    * 当前 CDN 对外访问域名，需要带协议
    * existCheck 检查，如果默认cdn域名无法访问，使用开放访问的 cdn 域名进行检查