feat(portal): 接入 scowd 文件分片上传 (#1346)

接入 scowd 分片上传相关接口 1. initMultipartUpload 接口，初始化分片上传，获取基本上传信息 2. 使用原本的上传接口上传分片 3. 分片上传完成后调用 mergeFileChunks 接口进行分片合并 scowd部分：PKUHPC/SCOWD#10
PKUHPC · Aug 13, 2024 · abd69cb · abd69cb
1 parent ac6805d
commit abd69cb
Show file tree

Hide file tree

Showing 26 changed files with 621 additions and 26 deletions.
diff --git a/.changeset/warm-gifts-know.md b/.changeset/warm-gifts-know.md
@@ -0,0 +1,11 @@
+---
+"@scow/portal-server": patch
+"@scow/lib-operation-log": patch
+"@scow/scowd-protos": patch
+"@scow/portal-web": patch
+"@scow/mis-web": patch
+"@scow/lib-server": patch
+"@scow/grpc-api": minor
+---
+
+接入 scowd 文件分片上传
diff --git a/apps/mis-web/src/i18n/en.ts b/apps/mis-web/src/i18n/en.ts
@@ -1244,6 +1244,8 @@ export default {
       customEvent: "Custom Operation Event",
       activateCluster: "Activate Cluster",
       deactivateCluster: "Deactivate Cluster",
+      mergeFileChunks: "Merge and upload temporary file blocks",
+      initMultipartUpload: "Initial multipart upload file",
     },
     operationDetails: {
       login: "User Login",

diff --git a/apps/mis-web/src/i18n/zh_cn.ts b/apps/mis-web/src/i18n/zh_cn.ts
@@ -1242,6 +1242,8 @@ export default {
       customEvent: "自定义操作行为",
       activateCluster: "启用集群",
       deactivateCluster: "停用集群",
+      mergeFileChunks: "合并临时文件块",
+      initMultipartUpload: "初始化分片上传",
     },
     operationDetails: {
       login: "用户登录",

diff --git a/apps/mis-web/src/models/operationLog.ts b/apps/mis-web/src/models/operationLog.ts
@@ -175,6 +175,8 @@ export const getOperationTypeTexts = (t: OperationTextsTransType): {[key in LibO
     activateCluster: t(pTypes("activateCluster")),
     deactivateCluster: t(pTypes("deactivateCluster")),
     customEvent: t(pTypes("customEvent")),
+    mergeFileChunks: t(pTypes("mergeFileChunks")),
+    initMultipartUpload: t(pTypes("initMultipartUpload")),
   };
 
 };
@@ -202,6 +204,8 @@ export const OperationCodeMap: {[key in LibOperationType]: string } = {
   moveFileItem: "010506",
   copyFileItem: "010507",
   submitFileItemAsJob: "010508",
+  mergeFileChunks: "010509",
+  initMultipartUpload: "010510",
   setJobTimeLimit: "010601",
   createImage:"010701",
   updateImage:"010702",

diff --git a/apps/portal-server/src/clusterops/file/index.ts b/apps/portal-server/src/clusterops/file/index.ts
@@ -25,15 +25,15 @@ export const fileOps = (cluster: string): FileOps => {
   const clusterInfo = configClusters[cluster];
   if (clusterInfo.scowd?.enabled) {
     const client = getScowdClient(cluster);
-    
+
     return {
       ...scowdFileServices(client),
     };
   } else {
     const host = getClusterLoginNode(cluster);
 
     if (!host) { throw clusterNotFound(cluster); }
-    
+
     return {
       ...sshFileServices(host),
     };

diff --git a/apps/portal-server/src/clusterops/file/scowdFile.ts b/apps/portal-server/src/clusterops/file/scowdFile.ts
@@ -110,7 +110,7 @@ export const scowdFileServices = (client: ScowdClient): FileOps => ({
           type: fileInfo_FileTypeFromJSON(info.fileType),
           mtime: info.modTime,
           mode: info.mode,
-          size: Number(info.size),
+          size: Number(info.sizeByte),
         };
       });
       return { results };
@@ -124,7 +124,7 @@ export const scowdFileServices = (client: ScowdClient): FileOps => ({
 
     try {
       const readStream = client.file.download({
-        userId, path, chunkSize: config.DOWNLOAD_CHUNK_SIZE,
+        userId, path, chunkSizeByte: config.DOWNLOAD_CHUNK_SIZE,
       });
 
       for await (const response of readStream) {
@@ -183,7 +183,7 @@ export const scowdFileServices = (client: ScowdClient): FileOps => ({
     try {
       const res = await client.file.getFileMetadata({ userId, filePath: path });
 
-      return { size: Number(res.size), type: res.type === FileType.DIR ? "dir" : "file" };
+      return { size: Number(res.sizeByte), type: res.type === FileType.DIR ? "dir" : "file" };
 
     } catch (err) {
       throw mapTRPCExceptionToGRPC(err);

diff --git a/apps/portal-server/src/services/file.ts b/apps/portal-server/src/services/file.ts
@@ -12,13 +12,17 @@
 
 import { plugin } from "@ddadaal/tsgrpc-server";
 import { ServiceError, status } from "@grpc/grpc-js";
+import { Status } from "@grpc/grpc-js/build/src/constants";
 import { loggedExec, sftpAppendFile, sftpExists, sftpMkdir,
   sftpReadFile, sftpRealPath, sshRmrf } from "@scow/lib-ssh";
-import { FileServiceServer, FileServiceService, TransferInfo } from "@scow/protos/build/portal/file";
+import {
+  FileInfo, fileInfo_FileTypeFromJSON, FileServiceServer, FileServiceService, TransferInfo,
+} from "@scow/protos/build/portal/file";
 import { getClusterOps } from "src/clusterops";
 import { configClusters } from "src/config/clusters";
 import { checkActivatedClusters } from "src/utils/clusters";
 import { clusterNotFound } from "src/utils/errors";
+import { getScowdClient, mapTRPCExceptionToGRPC } from "src/utils/scowd";
 import { getClusterLoginNode, getClusterTransferNode, sshConnect, tryGetClusterTransferNode } from "src/utils/ssh";
 
 export const fileServiceServer = plugin((server) => {
@@ -193,6 +197,78 @@ export const fileServiceServer = plugin((server) => {
 
     },
 
+    initMultipartUpload: async ({ request }) => {
+
+      const { cluster, userId, path, name } = request;
+      await checkActivatedClusters({ clusterIds: cluster });
+
+      const host = getClusterLoginNode(cluster);
+
+      if (!host) { throw clusterNotFound(cluster); }
+
+      const clusterInfo = configClusters[cluster];
+
+      if (!clusterInfo.scowd?.enabled) {
+        throw {
+          code: Status.UNIMPLEMENTED,
+          message: "To use this interface, you need to enable scowd.",
+        } as ServiceError;
+      }
+
+      const client = getScowdClient(cluster);
+
+      try {
+        const initData = await client.file.initMultipartUpload({ userId, path, name });
+
+        return [{
+          ...initData,
+          chunkSizeByte: Number(initData.chunkSizeByte),
+          filesInfo: initData.filesInfo.map((info): FileInfo => {
+            return {
+              name: info.name,
+              type: fileInfo_FileTypeFromJSON(info.fileType),
+              mtime: info.modTime,
+              mode: info.mode,
+              size: Number(info.sizeByte),
+            };
+          }),
+        }];
+
+      } catch (err) {
+        throw mapTRPCExceptionToGRPC(err);
+      }
+    },
+
+    mergeFileChunks: async ({ request }) => {
+      const { cluster, userId, path, name, sizeByte } = request;
+      await checkActivatedClusters({ clusterIds: cluster });
+
+      const host = getClusterLoginNode(cluster);
+
+      if (!host) { throw clusterNotFound(cluster); }
+
+      const clusterInfo = configClusters[cluster];
+
+      if (!clusterInfo.scowd?.enabled) {
+        throw {
+          code: Status.UNIMPLEMENTED,
+          message: "To use this interface, you need to enable scowd.",
+        } as ServiceError;
+      }
+
+      const client = getScowdClient(cluster);
+
+      try {
+        await client.file.mergeFileChunks({ userId, path, name, sizeByte: BigInt(sizeByte) });
+
+        return [{}];
+
+      } catch (err) {
+        throw mapTRPCExceptionToGRPC(err);
+      }
+    },
+
+
     getFileMetadata: async ({ request, logger }) => {
       const { userId, cluster, path } = request;
       await checkActivatedClusters({ clusterIds: cluster });

diff --git a/apps/portal-server/src/utils/scowd.ts b/apps/portal-server/src/utils/scowd.ts
@@ -29,6 +29,7 @@ export function getLoginNodeScowdUrl(cluster: string, host: string): string | un
   if (!loginNode) return undefined;
 
   const { address, scowdPort } = loginNode;
+
   return config.SCOWD_SSL_ENABLED
     ? `https://${removePort(address)}:${scowdPort}` : `http://${removePort(address)}:${scowdPort}`;
 }

diff --git a/apps/portal-web/package.json b/apps/portal-web/package.json
@@ -52,8 +52,11 @@
     "@sinclair/typebox": "0.32.34",
     "@uiw/codemirror-theme-github": "4.22.2",
     "@uiw/react-codemirror": "4.21.20",
+    "@xterm/addon-fit": "0.10.0",
+    "@xterm/xterm": "5.5.0",
     "antd": "5.18.3",
     "busboy": "1.6.0",
+    "crypto-js": "4.2.0",
     "dayjs": "1.11.11",
     "google-protobuf": "3.21.2",
     "http-proxy": "1.18.1",
@@ -63,6 +66,7 @@
     "next-compose-plugins": "2.2.1",
     "nookies": "2.5.2",
     "nprogress": "0.2.0",
+    "p-limit": "6.1.0",
     "react": "18.3.1",
     "react-async": "10.0.1",
     "react-dom": "18.3.1",
@@ -73,9 +77,7 @@
     "styled-components": "6.1.11",
     "tslib": "2.6.3",
     "typescript": "5.5.2",
-    "ws": "8.17.1",
-    "@xterm/xterm": "5.5.0",
-    "@xterm/addon-fit": "0.10.0"
+    "ws": "8.17.1"
   },
   "devDependencies": {
     "@ddadaal/next-typed-api-routes-cli": "0.9.1",

diff --git a/apps/portal-web/src/apis/api.mock.ts b/apps/portal-web/src/apis/api.mock.ts
@@ -315,6 +315,12 @@ export const mockApi: MockApi<typeof api> = {
     },
   }),
 
+  mergeFileChunks: null,
+  initMultipartUpload: async () => ({
+    tempFileDir: "home/user/scow/tempDir",
+    chunkSizeByte: 5 * 1024 * 1024,
+    filesInfo: [],
+  }),
   getClustersRuntimeInfo: async () => ({
     results: [{
       clusterId: "hpc01",

diff --git a/apps/portal-web/src/apis/api.ts b/apps/portal-web/src/apis/api.ts
@@ -44,8 +44,10 @@ import type { DownloadFileSchema } from "src/pages/api/file/download";
 import type { FileExistSchema } from "src/pages/api/file/fileExist";
 import type { GetFileTypeSchema } from "src/pages/api/file/getFileType";
 import type { GetHomeDirectorySchema } from "src/pages/api/file/getHome";
+import type { InitMultipartUploadSchema } from "src/pages/api/file/initMultipartUpload";
 import type { ListFileSchema } from "src/pages/api/file/list";
 import type { ListAvailableTransferClustersSchema } from "src/pages/api/file/listAvailableTransferClusters";
+import type { MergeFileChunksSchema } from "src/pages/api/file/mergeFileChunks";
 import type { MkdirSchema } from "src/pages/api/file/mkdir";
 import type { MoveFileItemSchema } from "src/pages/api/file/move";
 import type { QueryFileTransferProgressSchema } from "src/pages/api/file/queryFileTransferProgress";
@@ -97,8 +99,10 @@ export const api = {
   fileExist: apiClient.fromTypeboxRoute<typeof FileExistSchema>("GET", "/api/file/fileExist"),
   getFileType: apiClient.fromTypeboxRoute<typeof GetFileTypeSchema>("GET", "/api/file/getFileType"),
   getHomeDirectory: apiClient.fromTypeboxRoute<typeof GetHomeDirectorySchema>("GET", "/api/file/getHome"),
+  initMultipartUpload: apiClient.fromTypeboxRoute<typeof InitMultipartUploadSchema>("POST", "/api/file/initMultipartUpload"),
   listFile: apiClient.fromTypeboxRoute<typeof ListFileSchema>("GET", "/api/file/list"),
   listAvailableTransferClusters: apiClient.fromTypeboxRoute<typeof ListAvailableTransferClustersSchema>("GET", "/api/file/listAvailableTransferClusters"),
+  mergeFileChunks: apiClient.fromTypeboxRoute<typeof MergeFileChunksSchema>("POST", "/api/file/mergeFileChunks"),
   mkdir: apiClient.fromTypeboxRoute<typeof MkdirSchema>("POST", "/api/file/mkdir"),
   moveFileItem: apiClient.fromTypeboxRoute<typeof MoveFileItemSchema>("PATCH", "/api/file/move"),
   queryFileTransferProgress: apiClient.fromTypeboxRoute<typeof QueryFileTransferProgressSchema>("GET", "/api/file/queryFileTransferProgress"),

diff --git a/apps/portal-web/src/i18n/en.ts b/apps/portal-web/src/i18n/en.ts
@@ -343,6 +343,10 @@ export default {
         existedModalOk: "Confirm",
         dragText: "Click or drag files here",
         hintText: "Supports uploading single or multiple files",
+        multipartUploadError: "Upload file failed: {}",
+        calculateHashError: "Error calculating hash: {}",
+        uploadFileListNotExist: "The uploaded file list does not exist: {}",
+        mergeFileChunksErrorText: "Failed to merge file {}, please try again",
       },
     },
     // desktop

diff --git a/apps/portal-web/src/i18n/zh_cn.ts b/apps/portal-web/src/i18n/zh_cn.ts
@@ -343,6 +343,10 @@ export default {
         existedModalOk: "确认",
         dragText: "点击或者将文件拖动到这里",
         hintText: "支持上传单个或者多个文件",
+        multipartUploadError: "文件上传失败: {}",
+        calculateHashError: "计算哈希值错误: {}",
+        uploadFileListNotExist: "上传文件列表中不存在: {}",
+        mergeFileChunksErrorText: "合并文件 {} 失败，请重试",
       },
     },
     // desktop

diff --git a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx
@@ -54,6 +54,7 @@ interface Props {
   cluster: Cluster;
   path: string;
   urlPrefix: string;
+  scowdEnabled: boolean;
 }
 
 interface PromiseSettledResult {
@@ -93,7 +94,7 @@ interface Operation {
 
 const p = prefix("pageComp.fileManagerComp.fileManager.");
 
-export const FileManager: React.FC<Props> = ({ cluster, path, urlPrefix }) => {
+export const FileManager: React.FC<Props> = ({ cluster, path, urlPrefix, scowdEnabled }) => {
 
   const router = useRouter();
 
@@ -411,6 +412,7 @@ export const FileManager: React.FC<Props> = ({ cluster, path, urlPrefix }) => {
             cluster={cluster.id}
             path={path}
             reload={reload}
+            scowdEnabled={scowdEnabled}
           >
             {t(p("tableInfo.uploadButton"))}
           </UploadButton>