From a8d164fdeddadec86ef1431879e5cac0ecad6e84 Mon Sep 17 00:00:00 2001 From: picca Sun Date: Fri, 13 Oct 2023 11:59:33 +0000 Subject: [PATCH 01/13] impl --- apps/portal-server/src/services/job.ts | 34 ++++++ apps/portal-web/src/apis/api.mock.ts | 2 + apps/portal-web/src/apis/api.ts | 2 + apps/portal-web/src/i18n/en.ts | 4 + apps/portal-web/src/i18n/zh_cn.ts | 4 + .../filemanager/FileManager.tsx | 30 +++++ .../src/pages/api/job/submitFileAsJob.ts | 105 ++++++++++++++++++ dev/test-adapter/src/services/job.ts | 4 + libs/protos/scheduler-adapter/package.json | 2 +- protos/portal/job.proto | 15 +++ 10 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 apps/portal-web/src/pages/api/job/submitFileAsJob.ts diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index 066b6856dc..03c2e30f1b 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -218,6 +218,40 @@ export const jobServiceServer = plugin((server) => { return [{ jobId: reply.jobId }]; }, + submitFileAsJob: async ({ request }) => { + const { cluster, userId, fileDirectory } = request; + + const client = getAdapterClient(cluster); + if (!client) { throw clusterNotFound(cluster); } + + // make sure working directory exists + const host = getClusterLoginNode(cluster); + if (!host) { throw clusterNotFound(cluster); } + // await sshConnect(host, userId, logger, async (ssh) => { + // const sftp = await ssh.requestSFTP(); + // await createDirectoriesRecursively(sftp, workingDirectory); + // }); + + const reply = await asyncClientCall(client.job, "submitFileAsJob", { + userId, fileDirectory, + }).catch((e) => { + const ex = e as ServiceError; + const errors = parseErrorDetails(ex.metadata); + if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "SBATCH_FAILED") { + throw { + code: Status.INTERNAL, + message: "sbatch failed", + details: e.details, + }; + } else { + throw e; + } + }); + + return [{ jobId: reply.jobId }]; + }, + + }); }); diff --git a/apps/portal-web/src/apis/api.mock.ts b/apps/portal-web/src/apis/api.mock.ts index ca405de502..acc4974c4b 100644 --- a/apps/portal-web/src/apis/api.mock.ts +++ b/apps/portal-web/src/apis/api.mock.ts @@ -223,6 +223,8 @@ export const mockApi: MockApi = { submitJob: async () => ({ jobId: 10 }), + submitFileAsJob: async () => ({ jobId: 10 }), + getAppLastSubmission: async () => ({ lastSubmissionInfo: { userId: "test123", diff --git a/apps/portal-web/src/apis/api.ts b/apps/portal-web/src/apis/api.ts index e19351ed7c..edad8e6eb0 100644 --- a/apps/portal-web/src/apis/api.ts +++ b/apps/portal-web/src/apis/api.ts @@ -49,6 +49,7 @@ import type { GetJobTemplateSchema } from "src/pages/api/job/getJobTemplate"; import type { GetRunningJobsSchema } from "src/pages/api/job/getRunningJobs"; import type { ListJobTemplatesSchema } from "src/pages/api/job/listJobTemplates"; import type { RenameJobTemplateSchema } from "src/pages/api/job/renameJobTemplate"; +import { SubmitFileAsJobSchema } from "src/pages/api/job/submitFileAsJob"; import type { SubmitJobSchema } from "src/pages/api/job/submitJob"; import type { ChangePasswordSchema } from "src/pages/api/profile/changePassword"; import type { CheckPasswordSchema } from "src/pages/api/profile/checkPassword"; @@ -92,6 +93,7 @@ export const api = { listJobTemplates: apiClient.fromTypeboxRoute("GET", "/api/job/listJobTemplates"), renameJobTemplate: apiClient.fromTypeboxRoute("POST", "/api/job/renameJobTemplate"), submitJob: apiClient.fromTypeboxRoute("POST", "/api/job/submitJob"), + submitFileAsJob: apiClient.fromTypeboxRoute("POST", "/api/job/submitFileAsJob"), changePassword: apiClient.fromTypeboxRoute("PATCH", "/api/profile/changePassword"), checkPassword: apiClient.fromTypeboxRoute("GET", "/api/profile/checkPassword"), }; diff --git a/apps/portal-web/src/i18n/en.ts b/apps/portal-web/src/i18n/en.ts index e8071d3be7..f4c37855b4 100644 --- a/apps/portal-web/src/i18n/en.ts +++ b/apps/portal-web/src/i18n/en.ts @@ -234,6 +234,10 @@ export default { deleteConfirmContent: "Confirm deletion of {}?", deleteConfirmOk: "Confirm", deleteSuccessMessage: "Deleted successfully", + submitConfirmTitle: "Submit Confirmation", + submitConfirmContent: "Confirm submission of {} to {}?", + submitConfirmOk: "Confirm", + submitSuccessMessage: "Submitted Successfully", }, }, fileTable: { diff --git a/apps/portal-web/src/i18n/zh_cn.ts b/apps/portal-web/src/i18n/zh_cn.ts index 241e20274b..d33eee7643 100644 --- a/apps/portal-web/src/i18n/zh_cn.ts +++ b/apps/portal-web/src/i18n/zh_cn.ts @@ -234,6 +234,10 @@ export default { deleteConfirmContent: "确认删除{}?", deleteConfirmOk: "确认", deleteSuccessMessage: "删除成功", + submitConfirmTitle: "确认提交", + submitConfirmContent: "确认提交{}至{}?", + submitConfirmOk: "确认", + submitSuccessMessage: "删除成功", }, }, fileTable: { diff --git a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx index a157b7959b..528b026dba 100644 --- a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx @@ -506,6 +506,36 @@ export const FileManager: React.FC = ({ cluster, path, urlPrefix }) => { > {t("button.deleteButton")} + { + i.type === "FILE" ? ( + { + const fullPath = join(path, i.name); + modal.confirm({ + title: t(p("tableInfo.submitConfirmTitle")), + // icon: < />, + content: t(p("tableInfo.submitConfirmContent"), + [i.name, getI18nConfigCurrentText(cluster.name, languageId)]), + okText: t(p("tableInfo.submitConfirmOk")), + onOk: async () => { + await api.submitFileAsJob({ + body: { + cluster: cluster.id, + fileDirectory: fullPath, + }, + }) + .then(() => { + message.success(t(p("tableInfo.submitSuccessMessage"))); + resetSelectedAndOperation(); + reload(); + }); + }, + }); + }} + > + {t("button.submitButton")} + + ) : undefined + } )} /> diff --git a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts new file mode 100644 index 0000000000..e2cbec4bac --- /dev/null +++ b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts @@ -0,0 +1,105 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; +import { status } from "@grpc/grpc-js"; +import { JobServiceClient } from "@scow/protos/build/portal/job"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { OperationResult, OperationType } from "src/models/operationLog"; +import { callLog } from "src/server/operationLog"; +import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; +import { handlegRPCError, parseIp } from "src/utils/server"; + +export const SubmitFileAsJobSchema = typeboxRouteSchema({ + method: "POST", + + body: Type.Object({ + cluster: Type.String(), + // 文件绝对路径 + fileDirectory: Type.String(), + }), + + responses: { + 201: Type.Object({ + jobId: Type.Number(), + }), + + 400: Type.Object({ + message: Type.String(), + }), + + 500: Type.Object({ + code: Type.Literal("SCHEDULER_FAILED"), + message: Type.String(), + }), + }, +}); + +const auth = authenticate(() => true); + +export default route(SubmitFileAsJobSchema, async (req, res) => { + + const info = await auth(req, res); + + if (!info) { return; } + + const { cluster, fileDirectory } = req.body; + + const client = getClient(JobServiceClient); + + // TODO: log确认 + // const logInfo = { + // operatorUserId: info.identityId, + // operatorIp: parseIp(req) ?? "", + // operationTypePayload:{ + // accountName: account, + // }, + // }; + + return await asyncUnaryCall(client, "submitFileAsJob", { + cluster, userId: info.identityId, fileDirectory + , + }) + .then(async ({ jobId }) => { + // TODO: log确认 + // await callLog( + // { ...logInfo, + // operationTypeName: OperationType.submitJob, + // operationTypePayload: { ... logInfo.operationTypePayload, jobId } }, + // OperationResult.SUCCESS, + // ); + // if (save) { + // await callLog( + // { + // ...logInfo, + // operationTypeName: OperationType.addJobTemplate, + // operationTypePayload: { ... logInfo.operationTypePayload, jobTemplateId: `${jobName}-${jobId}` }, + // }, + // OperationResult.SUCCESS, + // ); + // } + return { 201: { jobId } } as const; + }) + .catch(handlegRPCError({ + [status.INTERNAL]: (err) => ({ 500: { code: "SCHEDULER_FAILED", message: err.details } } as const), + })); + // async () => await callLog( + // { ...logInfo, + // operationTypeName: OperationType.submitJob, + // operationTypePayload: { ... logInfo.operationTypePayload, jobId: -1 }, + // }, + // OperationResult.FAIL, + // ))); +}); diff --git a/dev/test-adapter/src/services/job.ts b/dev/test-adapter/src/services/job.ts index ccdc9d31be..ba8809fa1d 100644 --- a/dev/test-adapter/src/services/job.ts +++ b/dev/test-adapter/src/services/job.ts @@ -55,6 +55,10 @@ export const jobServiceServer = plugin((server) => { return [{ jobId: 1, generatedScript: "" }]; }, + submitFileAsJob: async () => { + return [{ jobId: 1 }]; + }, + cancelJob: async () => { return [{}]; }, diff --git a/libs/protos/scheduler-adapter/package.json b/libs/protos/scheduler-adapter/package.json index 7398cdaddd..db74c7aa33 100644 --- a/libs/protos/scheduler-adapter/package.json +++ b/libs/protos/scheduler-adapter/package.json @@ -5,7 +5,7 @@ "main": "build/index.js", "private": true, "scripts": { - "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git", + "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git#branch=add_job_submit_file_as_job_proto", "build": "rimraf build && tsc" }, "files": [ diff --git a/protos/portal/job.proto b/protos/portal/job.proto index 419381d19b..51c330886b 100644 --- a/protos/portal/job.proto +++ b/protos/portal/job.proto @@ -159,6 +159,19 @@ message SubmitJobResponse { uint32 job_id = 1; } +// fileDirectory: file's absolute path +message SubmitFileAsJobRequest { + string cluster = 1; + string user_id = 2; + string file_directory = 3; +} + +// NOT_FOUND: cluster is not found +// INTERNAL: error raised from scheduler. details has the error message +message SubmitFileAsJobResponse { + uint32 job_id = 1; +} + service JobService { rpc CancelJob(CancelJobRequest) returns (CancelJobResponse); @@ -177,4 +190,6 @@ service JobService { rpc RenameJobTemplate(RenameJobTemplateRequest) returns (RenameJobTemplateResponse); rpc SubmitJob(SubmitJobRequest) returns (SubmitJobResponse); + + rpc SubmitFileAsJob(SubmitFileAsJobRequest) returns (SubmitFileAsJobResponse); } From 5259ad9545b7d9b58e259b7aacc8af9675405b4e Mon Sep 17 00:00:00 2001 From: picca Sun Date: Tue, 17 Oct 2023 08:50:35 +0000 Subject: [PATCH 02/13] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E8=84=9A=E6=9C=AC=E7=9A=84=E6=93=8D=E4=BD=9C=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/mis-web/src/i18n/en.ts | 2 + apps/mis-web/src/i18n/zh_cn.ts | 2 + apps/mis-web/src/models/operationLog.ts | 5 ++ apps/portal-server/src/services/job.ts | 8 --- apps/portal-web/src/i18n/zh_cn.ts | 2 +- apps/portal-web/src/models/operationLog.ts | 1 + .../filemanager/FileManager.tsx | 4 +- .../src/pages/api/job/submitFileAsJob.ts | 56 ++++++++----------- protos/audit/operation_log.proto | 7 +++ 9 files changed, 43 insertions(+), 44 deletions(-) diff --git a/apps/mis-web/src/i18n/en.ts b/apps/mis-web/src/i18n/en.ts index 41e0a1ce60..27709c5502 100644 --- a/apps/mis-web/src/i18n/en.ts +++ b/apps/mis-web/src/i18n/en.ts @@ -992,6 +992,7 @@ export default { setPlatformBilling: "Set Platform Job Billing", createTenant: "Create Tenant", tenantPay: "Tenant Recharge", + submitFileItemAsJob: "Script Submission", }, operationDetails: { login: "User Login", @@ -1044,6 +1045,7 @@ export default { createTenant: "Create tenant {}, administrator: {}", tenantPay: "Recharge tenant {} by {} yuan", setPlatformBilling: "Set platform billing item {} price to {} yuan", + submitFileItemAsJob: "Cluster: {}, Submit Script: {}", }, }, userRoles: { diff --git a/apps/mis-web/src/i18n/zh_cn.ts b/apps/mis-web/src/i18n/zh_cn.ts index cf7ed953b4..ac0f82d38d 100644 --- a/apps/mis-web/src/i18n/zh_cn.ts +++ b/apps/mis-web/src/i18n/zh_cn.ts @@ -992,6 +992,7 @@ export default { setPlatformBilling: "设置平台作业计费", createTenant: "创建租户", tenantPay: "租户充值", + submitFileItemAsJob: "提交脚本", }, operationDetails: { login: "用户登录", @@ -1044,6 +1045,7 @@ export default { createTenant: "创建租户{}, 租户管理员为: {}", tenantPay: "为租户{}充值{}元", setPlatformBilling: "设置平台的计费项{}价格为{}元", + submitFileItemAsJob: "集群：{}，提交脚本：{}", }, }, userRoles: { diff --git a/apps/mis-web/src/models/operationLog.ts b/apps/mis-web/src/models/operationLog.ts index 4e2f94a0af..f631674414 100644 --- a/apps/mis-web/src/models/operationLog.ts +++ b/apps/mis-web/src/models/operationLog.ts @@ -76,6 +76,7 @@ export const OperationType: OperationTypeEnum = { setPlatformBilling: "setPlatformBilling", createTenant: "createTenant", tenantPay: "tenantPay", + submitFileItemAsJob: "submitFileItemAsJob", }; export const OperationLog = Type.Object({ @@ -163,6 +164,7 @@ export const getOperationTypeTexts = (t: OperationTextsTransType): { [key in Lib setPlatformBilling: t(pTypes("setPlatformBilling")), createTenant: t(pTypes("createTenant")), tenantPay: t(pTypes("tenantPay")), + submitFileItemAsJob: t(pTypes("submitFileItemAsJob")), }; }; @@ -186,6 +188,7 @@ export const OperationCodeMap: { [key in LibOperationType]: string } = { deleteDirectory: "010505", moveFileItem: "010506", copyFileItem: "010507", + submitFileItemAsJob: "010508", setJobTimeLimit: "010601", createUser: "020201", addUserToAccount: "020202", @@ -363,6 +366,8 @@ export const getOperationDetail = ( case "setPlatformBilling": return t(pDetails("setPlatformBilling"), [operationEvent[logEvent].path, nullableMoneyToString(operationEvent[logEvent].price)]); + case "submitFileItemAsJob": + return t(pDetails("submitFileItemAsJob"), [operationEvent[logEvent].clusterId, operationEvent[logEvent].path]); default: return "-"; } diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index 03c2e30f1b..c019c29bd3 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -224,14 +224,6 @@ export const jobServiceServer = plugin((server) => { const client = getAdapterClient(cluster); if (!client) { throw clusterNotFound(cluster); } - // make sure working directory exists - const host = getClusterLoginNode(cluster); - if (!host) { throw clusterNotFound(cluster); } - // await sshConnect(host, userId, logger, async (ssh) => { - // const sftp = await ssh.requestSFTP(); - // await createDirectoriesRecursively(sftp, workingDirectory); - // }); - const reply = await asyncClientCall(client.job, "submitFileAsJob", { userId, fileDirectory, }).catch((e) => { diff --git a/apps/portal-web/src/i18n/zh_cn.ts b/apps/portal-web/src/i18n/zh_cn.ts index d33eee7643..e6a0b0aeee 100644 --- a/apps/portal-web/src/i18n/zh_cn.ts +++ b/apps/portal-web/src/i18n/zh_cn.ts @@ -237,7 +237,7 @@ export default { submitConfirmTitle: "确认提交", submitConfirmContent: "确认提交{}至{}?", submitConfirmOk: "确认", - submitSuccessMessage: "删除成功", + submitSuccessMessage: "提交成功", }, }, fileTable: { diff --git a/apps/portal-web/src/models/operationLog.ts b/apps/portal-web/src/models/operationLog.ts index 01ee58379f..f86d2dfcdb 100644 --- a/apps/portal-web/src/models/operationLog.ts +++ b/apps/portal-web/src/models/operationLog.ts @@ -71,4 +71,5 @@ export const OperationType: OperationTypeEnum = { tenantPay: "tenantPay", blockAccount: "blockAccount", unblockAccount: "unblockAccount", + submitFileItemAsJob: "submitFileItemAsJob", }; diff --git a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx index 528b026dba..2596443b77 100644 --- a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx @@ -512,7 +512,6 @@ export const FileManager: React.FC = ({ cluster, path, urlPrefix }) => { const fullPath = join(path, i.name); modal.confirm({ title: t(p("tableInfo.submitConfirmTitle")), - // icon: < />, content: t(p("tableInfo.submitConfirmContent"), [i.name, getI18nConfigCurrentText(cluster.name, languageId)]), okText: t(p("tableInfo.submitConfirmOk")), @@ -523,6 +522,9 @@ export const FileManager: React.FC = ({ cluster, path, urlPrefix }) => { fileDirectory: fullPath, }, }) + .httpError(500, (e) => { + e.code === "SCHEDULER_FAILED" ? message.error(e.message) : (() => { throw e; })(); + }) .then(() => { message.success(t(p("tableInfo.submitSuccessMessage"))); resetSelectedAndOperation(); diff --git a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts index e2cbec4bac..4c33178c7d 100644 --- a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts +++ b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts @@ -59,47 +59,35 @@ export default route(SubmitFileAsJobSchema, async (req, res) => { const client = getClient(JobServiceClient); - // TODO: log确认 - // const logInfo = { - // operatorUserId: info.identityId, - // operatorIp: parseIp(req) ?? "", - // operationTypePayload:{ - // accountName: account, - // }, - // }; + const logInfo = { + operatorUserId: info.identityId, + operatorIp: parseIp(req) ?? "", + operationTypePayload:{ + clusterId: cluster, path: fileDirectory, + }, + }; return await asyncUnaryCall(client, "submitFileAsJob", { cluster, userId: info.identityId, fileDirectory , }) .then(async ({ jobId }) => { - // TODO: log确认 - // await callLog( - // { ...logInfo, - // operationTypeName: OperationType.submitJob, - // operationTypePayload: { ... logInfo.operationTypePayload, jobId } }, - // OperationResult.SUCCESS, - // ); - // if (save) { - // await callLog( - // { - // ...logInfo, - // operationTypeName: OperationType.addJobTemplate, - // operationTypePayload: { ... logInfo.operationTypePayload, jobTemplateId: `${jobName}-${jobId}` }, - // }, - // OperationResult.SUCCESS, - // ); - // } + await callLog( + { ...logInfo, + operationTypeName: OperationType.submitFileItemAsJob, + operationTypePayload: { ... logInfo.operationTypePayload } }, + OperationResult.SUCCESS, + ); return { 201: { jobId } } as const; }) .catch(handlegRPCError({ - [status.INTERNAL]: (err) => ({ 500: { code: "SCHEDULER_FAILED", message: err.details } } as const), - })); - // async () => await callLog( - // { ...logInfo, - // operationTypeName: OperationType.submitJob, - // operationTypePayload: { ... logInfo.operationTypePayload, jobId: -1 }, - // }, - // OperationResult.FAIL, - // ))); + [status.INTERNAL]: (err) => ({ 500: { code: "SCHEDULER_FAILED" as const, message: err.details } }), + }, + async () => await callLog( + { ...logInfo, + operationTypeName: OperationType.submitFileItemAsJob, + operationTypePayload: { ... logInfo.operationTypePayload }, + }, + OperationResult.FAIL, + ))); }); diff --git a/protos/audit/operation_log.proto b/protos/audit/operation_log.proto index 9094c69f95..b63e3eb0aa 100644 --- a/protos/audit/operation_log.proto +++ b/protos/audit/operation_log.proto @@ -111,6 +111,11 @@ message CopyFileItem { string to_path = 3; } +message SubmitFileItemAsJob { + string cluster_id = 1; + string path = 2; +} + message SetJobTimeLimit { string account_name = 1; uint32 job_id = 2; @@ -331,6 +336,7 @@ message CreateOperationLogRequest { SetPlatformBilling set_platform_billing = 50; CreateTenant create_tenant = 51; TenantPay tenant_pay = 52; + SubmitFileItemAsJob submit_file_item_as_job = 53; } } @@ -391,6 +397,7 @@ message OperationLog { SetPlatformBilling set_platform_billing = 52; CreateTenant create_tenant = 53; TenantPay tenant_pay = 54; + SubmitFileItemAsJob submit_file_item_as_job = 55; } } From d05c0ef02f212af3277e01a510b1124288bd362c Mon Sep 17 00:00:00 2001 From: picca Sun Date: Thu, 19 Oct 2023 09:00:03 +0000 Subject: [PATCH 03/13] =?UTF-8?q?=E4=B8=8E=E9=80=82=E9=85=8D=E5=99=A8?= =?UTF-8?q?=E8=81=94=E8=B0=83=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/cold-moons-bathe.md | 5 +++ .changeset/proud-seahorses-repeat.md | 8 ++++ apps/portal-server/src/services/job.ts | 45 +++++++++++++++++-- apps/portal-web/src/i18n/en.ts | 1 + apps/portal-web/src/i18n/zh_cn.ts | 1 + .../filemanager/FileManager.tsx | 13 +++++- .../src/pages/api/job/submitFileAsJob.ts | 14 ++++-- protos/portal/job.proto | 2 +- 8 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 .changeset/cold-moons-bathe.md create mode 100644 .changeset/proud-seahorses-repeat.md diff --git a/.changeset/cold-moons-bathe.md b/.changeset/cold-moons-bathe.md new file mode 100644 index 0000000000..ceb3569af3 --- /dev/null +++ b/.changeset/cold-moons-bathe.md @@ -0,0 +1,5 @@ +--- +"@scow/grpc-api": patch +--- + +新增 submitFileAsJob 接口，连接 slurm 调度器，直接提交文件 sbatch 执行 diff --git a/.changeset/proud-seahorses-repeat.md b/.changeset/proud-seahorses-repeat.md new file mode 100644 index 0000000000..7d53bdb95f --- /dev/null +++ b/.changeset/proud-seahorses-repeat.md @@ -0,0 +1,8 @@ +--- +"@scow/portal-server": patch +"@scow/test-adapter": patch +"@scow/portal-web": patch +"@scow/mis-web": patch +--- + +新增文件管理下提交任意文件直接作为作业文本 sbatch 执行的功能 diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index c019c29bd3..e329fdb36d 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -15,7 +15,7 @@ import { ServiceError } from "@ddadaal/tsgrpc-common"; import { plugin } from "@ddadaal/tsgrpc-server"; import { Status } from "@grpc/grpc-js/build/src/constants"; import { jobInfoToPortalJobInfo, jobInfoToRunningjob } from "@scow/lib-scheduler-adapter"; -import { createDirectoriesRecursively } from "@scow/lib-ssh"; +import { createDirectoriesRecursively, sftpReadFile, sftpStat } from "@scow/lib-ssh"; import { JobServiceServer, JobServiceService } from "@scow/protos/build/portal/job"; import { parseErrorDetails } from "@scow/rich-error-model"; import { getClusterOps } from "src/clusterops"; @@ -218,14 +218,51 @@ export const jobServiceServer = plugin((server) => { return [{ jobId: reply.jobId }]; }, - submitFileAsJob: async ({ request }) => { - const { cluster, userId, fileDirectory } = request; + submitFileAsJob: async ({ request, logger }) => { + const { cluster, userId, filePath } = request; const client = getAdapterClient(cluster); if (!client) { throw clusterNotFound(cluster); } + const host = getClusterLoginNode(cluster); + if (!host) { throw clusterNotFound(cluster); } + + const script = await sshConnect(host, userId, logger, async (ssh) => { + + const sftp = await ssh.requestSFTP(); + + // 判断文件操作权限 + const stat = await sftpStat(sftp)(filePath).catch((e) => { + logger.error(e, "stat %s as %s failed", filePath, userId); + throw { + code: Status.PERMISSION_DENIED, message: `${filePath} is not accessible`, + }; + }); + // 文件SIZE大于1M不能提交sbatch执行 + if (stat.size / (1024 * 1024) > 1) { + throw { + code: Status.INVALID_ARGUMENT, message: `${filePath} is too large. Maximum file size is 1M`, + }; + } + + const isTextFile = await ssh.exec("file", [filePath]).then((res) => { + return res.match(/text/); + }); + // 文件不是文本文件不能提交Sbatch执行 + if (!isTextFile) { + throw { + code: Status.INVALID_ARGUMENT, message: `${filePath} is not a text file`, + }; + } + + return await sftpReadFile(sftp)(filePath) + .then((buffer) => { + return buffer.toString("utf-8"); + }); + }); + const reply = await asyncClientCall(client.job, "submitFileAsJob", { - userId, fileDirectory, + userId, script, }).catch((e) => { const ex = e as ServiceError; const errors = parseErrorDetails(ex.metadata); diff --git a/apps/portal-web/src/i18n/en.ts b/apps/portal-web/src/i18n/en.ts index f4c37855b4..fbb03546b8 100644 --- a/apps/portal-web/src/i18n/en.ts +++ b/apps/portal-web/src/i18n/en.ts @@ -238,6 +238,7 @@ export default { submitConfirmContent: "Confirm submission of {} to {}?", submitConfirmOk: "Confirm", submitSuccessMessage: "Submitted Successfully", + submitFailedMessage: "Submitted Failed", }, }, fileTable: { diff --git a/apps/portal-web/src/i18n/zh_cn.ts b/apps/portal-web/src/i18n/zh_cn.ts index e6a0b0aeee..e87a82cf3e 100644 --- a/apps/portal-web/src/i18n/zh_cn.ts +++ b/apps/portal-web/src/i18n/zh_cn.ts @@ -238,6 +238,7 @@ export default { submitConfirmContent: "确认提交{}至{}?", submitConfirmOk: "确认", submitSuccessMessage: "提交成功", + submitFailedMessage: "提交失败", }, }, fileTable: { diff --git a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx index 2596443b77..aa1c4d2a1b 100644 --- a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx @@ -519,11 +519,20 @@ export const FileManager: React.FC = ({ cluster, path, urlPrefix }) => { await api.submitFileAsJob({ body: { cluster: cluster.id, - fileDirectory: fullPath, + filePath: fullPath, }, }) .httpError(500, (e) => { - e.code === "SCHEDULER_FAILED" ? message.error(e.message) : (() => { throw e; })(); + e.code === "SCHEDULER_FAILED" ? modal.error({ + title: t(p("tableInfo.submitFailedMessage")), + content: e.message, + }) : (() => { throw e; })(); + }) + .httpError(400, (e) => { + e.code === "INVALID_ARGUMENT" || e.code === "INVALID_PATH" ? modal.error({ + title: t(p("tableInfo.submitFailedMessage")), + content: e.message, + }) : (() => { throw e; })(); }) .then(() => { message.success(t(p("tableInfo.submitSuccessMessage"))); diff --git a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts index 4c33178c7d..83ce3e1534 100644 --- a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts +++ b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts @@ -28,7 +28,7 @@ export const SubmitFileAsJobSchema = typeboxRouteSchema({ body: Type.Object({ cluster: Type.String(), // 文件绝对路径 - fileDirectory: Type.String(), + filePath: Type.String(), }), responses: { @@ -37,6 +37,10 @@ export const SubmitFileAsJobSchema = typeboxRouteSchema({ }), 400: Type.Object({ + code: Type.Union([ + Type.Literal("INVALID_ARGUMENT"), + Type.Literal("INVALID_PATH"), + ]), message: Type.String(), }), @@ -55,7 +59,7 @@ export default route(SubmitFileAsJobSchema, async (req, res) => { if (!info) { return; } - const { cluster, fileDirectory } = req.body; + const { cluster, filePath } = req.body; const client = getClient(JobServiceClient); @@ -63,12 +67,12 @@ export default route(SubmitFileAsJobSchema, async (req, res) => { operatorUserId: info.identityId, operatorIp: parseIp(req) ?? "", operationTypePayload:{ - clusterId: cluster, path: fileDirectory, + clusterId: cluster, path: filePath, }, }; return await asyncUnaryCall(client, "submitFileAsJob", { - cluster, userId: info.identityId, fileDirectory + cluster, userId: info.identityId, filePath , }) .then(async ({ jobId }) => { @@ -82,6 +86,8 @@ export default route(SubmitFileAsJobSchema, async (req, res) => { }) .catch(handlegRPCError({ [status.INTERNAL]: (err) => ({ 500: { code: "SCHEDULER_FAILED" as const, message: err.details } }), + [status.INVALID_ARGUMENT]: (err) => ({ 400: { code: "INVALID_ARGUMENT" as const, message: err.details } }), + [status.PERMISSION_DENIED]: (err) => ({ 400: { code: "INVALID_PATH" as const, message: err.details } }), }, async () => await callLog( { ...logInfo, diff --git a/protos/portal/job.proto b/protos/portal/job.proto index 51c330886b..00760a685e 100644 --- a/protos/portal/job.proto +++ b/protos/portal/job.proto @@ -163,7 +163,7 @@ message SubmitJobResponse { message SubmitFileAsJobRequest { string cluster = 1; string user_id = 2; - string file_directory = 3; + string file_path = 3; } // NOT_FOUND: cluster is not found From c4d25966ec685db4a9fc1bcf4a2d5e21d61d6c78 Mon Sep 17 00:00:00 2001 From: picca Sun Date: Thu, 19 Oct 2023 16:44:40 +0000 Subject: [PATCH 04/13] =?UTF-8?q?changset=E6=96=87=E5=AD=97=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/cold-moons-bathe.md | 2 +- .changeset/proud-seahorses-repeat.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/cold-moons-bathe.md b/.changeset/cold-moons-bathe.md index ceb3569af3..1f73dfffaf 100644 --- a/.changeset/cold-moons-bathe.md +++ b/.changeset/cold-moons-bathe.md @@ -2,4 +2,4 @@ "@scow/grpc-api": patch --- -新增 submitFileAsJob 接口，连接 slurm 调度器，直接提交文件 sbatch 执行 +新增 submitFileAsJob 接口，直接把文件作为作业提交调度器执行 diff --git a/.changeset/proud-seahorses-repeat.md b/.changeset/proud-seahorses-repeat.md index 7d53bdb95f..49cadf41e1 100644 --- a/.changeset/proud-seahorses-repeat.md +++ b/.changeset/proud-seahorses-repeat.md @@ -5,4 +5,4 @@ "@scow/mis-web": patch --- -新增文件管理下提交任意文件直接作为作业文本 sbatch 执行的功能 +在门户系统的文件管理下，新增将文件直接作为作业文本提交调度器执行的功能 From e4cb4c5044372ac8418405a9631d6b062b9ba074 Mon Sep 17 00:00:00 2001 From: picca Sun Date: Thu, 26 Oct 2023 08:06:24 +0000 Subject: [PATCH 05/13] =?UTF-8?q?=E5=A2=9E=E5=8A=A0jobId=E5=88=B0=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E6=88=90=E5=8A=9F=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/portal-web/src/i18n/en.ts | 2 +- apps/portal-web/src/i18n/zh_cn.ts | 2 +- .../src/pageComponents/filemanager/FileManager.tsx | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/portal-web/src/i18n/en.ts b/apps/portal-web/src/i18n/en.ts index fbb03546b8..9987d744ef 100644 --- a/apps/portal-web/src/i18n/en.ts +++ b/apps/portal-web/src/i18n/en.ts @@ -237,7 +237,7 @@ export default { submitConfirmTitle: "Submit Confirmation", submitConfirmContent: "Confirm submission of {} to {}?", submitConfirmOk: "Confirm", - submitSuccessMessage: "Submitted Successfully", + submitSuccessMessage: "Submitted successfully! Your new job ID is: {}", submitFailedMessage: "Submitted Failed", }, }, diff --git a/apps/portal-web/src/i18n/zh_cn.ts b/apps/portal-web/src/i18n/zh_cn.ts index e87a82cf3e..4298abf655 100644 --- a/apps/portal-web/src/i18n/zh_cn.ts +++ b/apps/portal-web/src/i18n/zh_cn.ts @@ -237,7 +237,7 @@ export default { submitConfirmTitle: "确认提交", submitConfirmContent: "确认提交{}至{}?", submitConfirmOk: "确认", - submitSuccessMessage: "提交成功", + submitSuccessMessage: "提交成功！您的新作业ID为：{}", submitFailedMessage: "提交失败", }, }, diff --git a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx index aa1c4d2a1b..7a063719b8 100644 --- a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx @@ -523,6 +523,7 @@ export const FileManager: React.FC = ({ cluster, path, urlPrefix }) => { }, }) .httpError(500, (e) => { + console.log("eeeeeee", e); e.code === "SCHEDULER_FAILED" ? modal.error({ title: t(p("tableInfo.submitFailedMessage")), content: e.message, @@ -534,8 +535,8 @@ export const FileManager: React.FC = ({ cluster, path, urlPrefix }) => { content: e.message, }) : (() => { throw e; })(); }) - .then(() => { - message.success(t(p("tableInfo.submitSuccessMessage"))); + .then((result) => { + message.success(t(p("tableInfo.submitSuccessMessage"), [result.jobId])); resetSelectedAndOperation(); reload(); }); From eed83954f2b0581088c03fce69d70a0d3bd2d2d6 Mon Sep 17 00:00:00 2001 From: picca Sun Date: Thu, 26 Oct 2023 09:37:11 +0000 Subject: [PATCH 06/13] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E5=9B=9E=E8=B0=83?= =?UTF-8?q?=E5=BA=A6=E5=99=A8=E9=80=82=E9=85=8D=E5=99=A8=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E5=9C=B0=E5=9D=80=E4=B8=BA=E4=B8=BB=E5=88=86?= =?UTF-8?q?=E6=94=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libs/protos/scheduler-adapter/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/protos/scheduler-adapter/package.json b/libs/protos/scheduler-adapter/package.json index db74c7aa33..7398cdaddd 100644 --- a/libs/protos/scheduler-adapter/package.json +++ b/libs/protos/scheduler-adapter/package.json @@ -5,7 +5,7 @@ "main": "build/index.js", "private": true, "scripts": { - "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git#branch=add_job_submit_file_as_job_proto", + "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git", "build": "rimraf build && tsc" }, "files": [ From 4160c66d2f3654745332cbe421e759cbed4f7e58 Mon Sep 17 00:00:00 2001 From: picca Sun Date: Fri, 3 Nov 2023 08:06:53 +0000 Subject: [PATCH 07/13] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=80=82=E9=85=8D?= =?UTF-8?q?=E5=99=A8=E6=8E=A5=E5=8F=A3=EF=BC=8C=E8=81=94=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/portal-server/src/services/job.ts | 2 +- dev/test-adapter/src/services/job.ts | 2 +- protos/portal/job.proto | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index e329fdb36d..25ad77dc49 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -261,7 +261,7 @@ export const jobServiceServer = plugin((server) => { }); }); - const reply = await asyncClientCall(client.job, "submitFileAsJob", { + const reply = await asyncClientCall(client.job, "submitScriptAsJob", { userId, script, }).catch((e) => { const ex = e as ServiceError; diff --git a/dev/test-adapter/src/services/job.ts b/dev/test-adapter/src/services/job.ts index ba8809fa1d..486d60f6f3 100644 --- a/dev/test-adapter/src/services/job.ts +++ b/dev/test-adapter/src/services/job.ts @@ -55,7 +55,7 @@ export const jobServiceServer = plugin((server) => { return [{ jobId: 1, generatedScript: "" }]; }, - submitFileAsJob: async () => { + submitScriptAsJob: async () => { return [{ jobId: 1 }]; }, diff --git a/protos/portal/job.proto b/protos/portal/job.proto index 00760a685e..16fb669625 100644 --- a/protos/portal/job.proto +++ b/protos/portal/job.proto @@ -159,7 +159,7 @@ message SubmitJobResponse { uint32 job_id = 1; } -// fileDirectory: file's absolute path +// filePath: file's absolute path message SubmitFileAsJobRequest { string cluster = 1; string user_id = 2; From 9243f38fdad7764be6221e7569845a44e791db10 Mon Sep 17 00:00:00 2001 From: picca Sun Date: Fri, 3 Nov 2023 08:09:25 +0000 Subject: [PATCH 08/13] =?UTF-8?q?=E4=BF=AE=E6=94=B9grpc-api=E7=9A=84change?= =?UTF-8?q?set=EF=BC=8C=E6=8F=90=E9=AB=98MINOR=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/cold-moons-bathe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cold-moons-bathe.md b/.changeset/cold-moons-bathe.md index 1f73dfffaf..c70fd7d1f4 100644 --- a/.changeset/cold-moons-bathe.md +++ b/.changeset/cold-moons-bathe.md @@ -1,5 +1,5 @@ --- -"@scow/grpc-api": patch +"@scow/grpc-api": minor --- 新增 submitFileAsJob 接口，直接把文件作为作业提交调度器执行 From 65b65b2230208b6cc32193b81a34161f36a344c1 Mon Sep 17 00:00:00 2001 From: picca Sun Date: Fri, 3 Nov 2023 09:00:44 +0000 Subject: [PATCH 09/13] =?UTF-8?q?=E5=90=88=E5=B9=B6master=EF=BC=8C?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=B0=83=E5=BA=A6=E5=99=A8=E9=80=82=E9=85=8D?= =?UTF-8?q?=E5=99=A8=E6=8E=A5=E5=8F=A3=E4=BB=93=E5=BA=93=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=9A=82=E4=B8=BA=E6=B5=8B=E8=AF=95=E7=94=A8=E4=B8=BB=E5=88=86?= =?UTF-8?q?=E6=94=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libs/protos/scheduler-adapter/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/protos/scheduler-adapter/package.json b/libs/protos/scheduler-adapter/package.json index 20cae6485e..70be99c9a0 100644 --- a/libs/protos/scheduler-adapter/package.json +++ b/libs/protos/scheduler-adapter/package.json @@ -5,7 +5,7 @@ "main": "build/index.js", "private": true, "scripts": { - "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git#tag=v1.0.1", + "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git", "build": "rimraf build && tsc" }, "files": [ From c43de761f349ac78ac5c9261c3bcede785c4ebcc Mon Sep 17 00:00:00 2001 From: picca Sun Date: Mon, 6 Nov 2023 01:38:26 +0000 Subject: [PATCH 10/13] =?UTF-8?q?=E4=BF=AE=E6=94=B9scow-scheduler-adapter-?= =?UTF-8?q?interface=E7=89=88=E6=9C=AC=E5=8F=B7=E5=88=B0v1.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libs/protos/scheduler-adapter/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/protos/scheduler-adapter/package.json b/libs/protos/scheduler-adapter/package.json index 70be99c9a0..9aa3b96b04 100644 --- a/libs/protos/scheduler-adapter/package.json +++ b/libs/protos/scheduler-adapter/package.json @@ -5,7 +5,7 @@ "main": "build/index.js", "private": true, "scripts": { - "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git", + "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git#tag=v1.2.0", "build": "rimraf build && tsc" }, "files": [ From 243a173c67f76c5bdb9bc1c2f3367a7f6c58cdbe Mon Sep 17 00:00:00 2001 From: picca Sun Date: Thu, 9 Nov 2023 09:19:48 +0000 Subject: [PATCH 11/13] add check version --- apps/portal-server/src/services/job.ts | 14 +++++- apps/portal-server/src/utils/clusters.ts | 49 ++++++++++++++++++ .../filemanager/FileManager.tsx | 3 +- .../src/pages/api/job/submitFileAsJob.ts | 8 ++- dev/test-adapter/src/app.ts | 2 + dev/test-adapter/src/services/version.ts | 25 ++++++++++ libs/scheduler-adapter/src/client.ts | 3 ++ libs/utils/src/version.ts | 50 +++++++++++++++++++ 8 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 dev/test-adapter/src/services/version.ts diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index 25ad77dc49..c4753e37b7 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -20,7 +20,7 @@ import { JobServiceServer, JobServiceService } from "@scow/protos/build/portal/j import { parseErrorDetails } from "@scow/rich-error-model"; import { getClusterOps } from "src/clusterops"; import { JobTemplate } from "src/clusterops/api/job"; -import { getAdapterClient } from "src/utils/clusters"; +import { checkSchedulerApiVersion, getAdapterClient } from "src/utils/clusters"; import { clusterNotFound } from "src/utils/errors"; import { getClusterLoginNode, sshConnect } from "src/utils/ssh"; @@ -218,12 +218,16 @@ export const jobServiceServer = plugin((server) => { return [{ jobId: reply.jobId }]; }, + submitFileAsJob: async ({ request, logger }) => { const { cluster, userId, filePath } = request; const client = getAdapterClient(cluster); if (!client) { throw clusterNotFound(cluster); } + // 检验scow与调度器的API版本是否一致 + await checkSchedulerApiVersion(client); + const host = getClusterLoginNode(cluster); if (!host) { throw clusterNotFound(cluster); } @@ -272,6 +276,14 @@ export const jobServiceServer = plugin((server) => { message: "sbatch failed", details: e.details, }; + } else if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED") { + throw { + code: Status.FAILED_PRECONDITION, + message: "precondition failed", + details: "The method submitScriptAsJob is not supported with your current scheduler adapter version. " + + "To use this method, you must upgrade to a scheduler adapter " + + "that complies with the 1.2.0 interface standard requirements or higher.", + }; } else { throw e; } diff --git a/apps/portal-server/src/utils/clusters.ts b/apps/portal-server/src/utils/clusters.ts index 4ba6cdc40d..193960a90c 100644 --- a/apps/portal-server/src/utils/clusters.ts +++ b/apps/portal-server/src/utils/clusters.ts @@ -10,9 +10,14 @@ * See the Mulan PSL v2 for more details. */ +import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { ServiceError, status } from "@grpc/grpc-js"; import { getSchedulerAdapterClient, SchedulerAdapterClient } from "@scow/lib-scheduler-adapter"; +import { parseErrorDetails } from "@scow/rich-error-model"; +import { ApiVersion, compareSemVersion, getCurrentScowSchedulerApiVersion } from "@scow/utils/build/version"; import { clusters } from "src/config/clusters"; +import { logger } from "./logger"; const adapterClientForClusters = Object.entries(clusters).reduce((prev, [cluster, c]) => { const client = getSchedulerAdapterClient(c.adapterUrl); @@ -23,3 +28,47 @@ const adapterClientForClusters = Object.entries(clusters).reduce((prev, [cluster export const getAdapterClient = (cluster: string) => { return adapterClientForClusters[cluster]; }; + +/** + * 判断当前集群下的调度器API版本是否已过时 + * @param client + * @returns + */ +export async function checkSchedulerApiVersion(client: SchedulerAdapterClient): Promise { + + let scheduleApiVersion: ApiVersion | null; + try { + scheduleApiVersion = await asyncClientCall(client.version, "getVersion", {}); + } catch (e) { + const ex = e as ServiceError; + const errors = parseErrorDetails(ex.metadata); + // 如果接口不存在 + if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED") { + scheduleApiVersion = { major: 1, minor: 1, patch: 0 }; + // 如果接口服务不存在 + } else if ((e as any).code === status.UNIMPLEMENTED) { + scheduleApiVersion = { major: 1, minor: 1, patch: 0 }; + } else { + scheduleApiVersion = null; + logger.warn( + "The scheduler API version can not be confirmed. Some functionalities may not operate as expected."); + } + } + const scowSchedulerApiVersion = getCurrentScowSchedulerApiVersion(); + + if (!scowSchedulerApiVersion && scheduleApiVersion) { + logger.warn("The current scow scheduler API version can not be confirmed. Please ensure you are using " + + "an API version that is compatible with the scheduler Api Version " + + `${scheduleApiVersion.major}.${scheduleApiVersion.minor}.${scheduleApiVersion.patch}.`); + } + + if (scowSchedulerApiVersion && scheduleApiVersion) { + const compareResult = compareSemVersion(scowSchedulerApiVersion, scheduleApiVersion); + if (compareResult === 1) { + logger.warn("The current scheduler API version is outdated. Some functionalities may not operate as expected. " + + "Please upgrade to version " + + `${scowSchedulerApiVersion.major}.${scowSchedulerApiVersion.minor}.${scowSchedulerApiVersion.patch} or later.`); + } + } + +}; diff --git a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx index 06f531fb9f..064b7a4ea0 100644 --- a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx @@ -523,8 +523,7 @@ export const FileManager: React.FC = ({ cluster, path, urlPrefix }) => { }, }) .httpError(500, (e) => { - console.log("eeeeeee", e); - e.code === "SCHEDULER_FAILED" ? modal.error({ + e.code === "SCHEDULER_FAILED" || e.code === "FAILED_PRECONDITION" ? modal.error({ title: t(p("tableInfo.submitFailedMessage")), content: e.message, }) : (() => { throw e; })(); diff --git a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts index 83ce3e1534..49e95f4b06 100644 --- a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts +++ b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts @@ -45,7 +45,10 @@ export const SubmitFileAsJobSchema = typeboxRouteSchema({ }), 500: Type.Object({ - code: Type.Literal("SCHEDULER_FAILED"), + code: Type.Union([ + Type.Literal("SCHEDULER_FAILED"), + Type.Literal("FAILED_PRECONDITION"), + ]), message: Type.String(), }), }, @@ -86,6 +89,9 @@ export default route(SubmitFileAsJobSchema, async (req, res) => { }) .catch(handlegRPCError({ [status.INTERNAL]: (err) => ({ 500: { code: "SCHEDULER_FAILED" as const, message: err.details } }), + [status.FAILED_PRECONDITION]: () => ({ 500: { + code: "FAILED_PRECONDITION" as const, + message: "The method submitScriptAsJob is not supported with your current scheduler adapter version." } }), [status.INVALID_ARGUMENT]: (err) => ({ 400: { code: "INVALID_ARGUMENT" as const, message: err.details } }), [status.PERMISSION_DENIED]: (err) => ({ 400: { code: "INVALID_PATH" as const, message: err.details } }), }, diff --git a/dev/test-adapter/src/app.ts b/dev/test-adapter/src/app.ts index 2b81ef8ec5..d7be925dfa 100644 --- a/dev/test-adapter/src/app.ts +++ b/dev/test-adapter/src/app.ts @@ -16,6 +16,7 @@ import { accountServiceServer } from "src/services/account"; import { configServiceServer } from "src/services/config"; import { jobServiceServer } from "src/services/job"; import { userServiceServer } from "src/services/user"; +import { versionServiceServer } from "src/services/version"; export async function createServer() { @@ -35,6 +36,7 @@ export async function createServer() { await server.register(userServiceServer); await server.register(jobServiceServer); await server.register(configServiceServer); + await server.register(versionServiceServer); return server; diff --git a/dev/test-adapter/src/services/version.ts b/dev/test-adapter/src/services/version.ts new file mode 100644 index 0000000000..e4931d8dc7 --- /dev/null +++ b/dev/test-adapter/src/services/version.ts @@ -0,0 +1,25 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { plugin } from "@ddadaal/tsgrpc-server"; +import { VersionServiceServer, VersionServiceService } from "@scow/scheduler-adapter-protos/build/protos/version"; + +export const versionServiceServer = plugin((server) => { + server.addService(VersionServiceService, { + + getVersion: async () => { + return [{ major: 0, minor: 0, patch: 0 }]; + }, + + }); + +}); diff --git a/libs/scheduler-adapter/src/client.ts b/libs/scheduler-adapter/src/client.ts index dd9587497c..0baafc76dd 100644 --- a/libs/scheduler-adapter/src/client.ts +++ b/libs/scheduler-adapter/src/client.ts @@ -15,6 +15,7 @@ import { AccountServiceClient } from "@scow/scheduler-adapter-protos/build/proto import { ConfigServiceClient } from "@scow/scheduler-adapter-protos/build/protos/config"; import { JobServiceClient } from "@scow/scheduler-adapter-protos/build/protos/job"; import { UserServiceClient } from "@scow/scheduler-adapter-protos/build/protos/user"; +import { VersionServiceClient } from "@scow/scheduler-adapter-protos/build/protos/version"; type ClientConstructor = new (address: string, credentials: ChannelCredentials) => TClient; @@ -24,6 +25,7 @@ export interface SchedulerAdapterClient { user: UserServiceClient; job: JobServiceClient; config: ConfigServiceClient; + version: VersionServiceClient; } export function getClient( @@ -41,5 +43,6 @@ export const getSchedulerAdapterClient = (address: string) => { user: getClient(address, UserServiceClient), job: getClient(address, JobServiceClient), config: getClient(address, ConfigServiceClient), + version: getClient(address, VersionServiceClient), }; }; diff --git a/libs/utils/src/version.ts b/libs/utils/src/version.ts index 46ff848ee8..d113a526b3 100644 --- a/libs/utils/src/version.ts +++ b/libs/utils/src/version.ts @@ -11,6 +11,7 @@ */ import { existsSync, readFileSync } from "fs"; +import { join } from "path"; interface VersionJsonInfo { tag?: string; @@ -28,3 +29,52 @@ export function readVersionFile(versionJsonFileName = "version.json") { return jsonInfo; } +// SemVer类型version +export type ApiVersion = { + major: number; + minor: number; + patch: number; +} + +// 获取当前调度器适配器接口仓库的API版本号 +export function getCurrentScowSchedulerApiVersion(): ApiVersion | null { + + const schedulerAdapterJsonFilePath = join(__dirname, "../../protos/scheduler-adapter/package.json"); + const packageJsonContent = JSON.parse(readFileSync(schedulerAdapterJsonFilePath, "utf-8")); + + const match = packageJsonContent.scripts.generate.match(/(?<=#tag=v)([\d.]+)/); + + const version = match ? match[1] : undefined; + + if (version) { + const versionParts = version.split("."); + return { + major: parseInt(versionParts[0]), + minor: parseInt(versionParts[1]), + patch: parseInt(versionParts[2]), + }; + } else { + return null; + } +} + +/** + * 比较Version1与Version2版本 + * @param version1 + * @param version2 + * @returns + * 1： Version1高于Version2 + * -1：Version1低于Version2 + * 0: Version1与Version2版本相同 + */ +export function compareSemVersion(version1: ApiVersion, version2: ApiVersion): number { + if (version1.major !== version2.major) { + return version1.major > version2.major ? 1 : -1; + } else if (version1.minor !== version2.minor) { + return version1.minor > version2.minor ? 1 : -1; + } else if (version1.patch !== version2.patch) { + return version1.patch > version2.patch ? 1 : -1; + } else { + return 0; + } +}; From a6036bb1f0924a02fcb3c096c3d7477df528b772 Mon Sep 17 00:00:00 2001 From: picca Sun Date: Thu, 9 Nov 2023 09:52:57 +0000 Subject: [PATCH 12/13] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=BA=E6=AF=94?= =?UTF-8?q?=E8=BE=83=E5=BD=93=E5=89=8D=E6=8E=A5=E5=8F=A3=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/portal-server/src/services/job.ts | 11 ++-- apps/portal-server/src/utils/clusters.ts | 81 +++++++++++++++++++----- libs/utils/src/version.ts | 36 +++++------ 3 files changed, 90 insertions(+), 38 deletions(-) diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index c4753e37b7..d0f58da6d6 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -18,6 +18,7 @@ import { jobInfoToPortalJobInfo, jobInfoToRunningjob } from "@scow/lib-scheduler import { createDirectoriesRecursively, sftpReadFile, sftpStat } from "@scow/lib-ssh"; import { JobServiceServer, JobServiceService } from "@scow/protos/build/portal/job"; import { parseErrorDetails } from "@scow/rich-error-model"; +import { ApiVersion } from "@scow/utils/build/version"; import { getClusterOps } from "src/clusterops"; import { JobTemplate } from "src/clusterops/api/job"; import { checkSchedulerApiVersion, getAdapterClient } from "src/utils/clusters"; @@ -225,8 +226,9 @@ export const jobServiceServer = plugin((server) => { const client = getAdapterClient(cluster); if (!client) { throw clusterNotFound(cluster); } + const interfaceApiVersion: ApiVersion = { major: 1, minor: 2, patch: 0 }; // 检验scow与调度器的API版本是否一致 - await checkSchedulerApiVersion(client); + await checkSchedulerApiVersion(client, interfaceApiVersion); const host = getClusterLoginNode(cluster); if (!host) { throw clusterNotFound(cluster); } @@ -280,9 +282,10 @@ export const jobServiceServer = plugin((server) => { throw { code: Status.FAILED_PRECONDITION, message: "precondition failed", - details: "The method submitScriptAsJob is not supported with your current scheduler adapter version. " - + "To use this method, you must upgrade to a scheduler adapter " - + "that complies with the 1.2.0 interface standard requirements or higher.", + details: "The method is not supported with the current scheduler adapter version. " + + "To use this method, the scheduler adapter must be upgraded to the version " + + `${interfaceApiVersion.major}.${interfaceApiVersion.minor}.${interfaceApiVersion.patch}` + + "or higher.", }; } else { throw e; diff --git a/apps/portal-server/src/utils/clusters.ts b/apps/portal-server/src/utils/clusters.ts index 193960a90c..c8f56c83c7 100644 --- a/apps/portal-server/src/utils/clusters.ts +++ b/apps/portal-server/src/utils/clusters.ts @@ -12,9 +12,10 @@ import { asyncClientCall } from "@ddadaal/tsgrpc-client"; import { ServiceError, status } from "@grpc/grpc-js"; +import { Status } from "@grpc/grpc-js/build/src/constants"; import { getSchedulerAdapterClient, SchedulerAdapterClient } from "@scow/lib-scheduler-adapter"; import { parseErrorDetails } from "@scow/rich-error-model"; -import { ApiVersion, compareSemVersion, getCurrentScowSchedulerApiVersion } from "@scow/utils/build/version"; +import { ApiVersion, compareSemVersion } from "@scow/utils/build/version"; import { clusters } from "src/config/clusters"; import { logger } from "./logger"; @@ -29,12 +30,62 @@ export const getAdapterClient = (cluster: string) => { return adapterClientForClusters[cluster]; }; + +// /** +// * 判断当前集群下的API版本与调度器API版本 +// * @param client +// * @returns +// */ +// export async function checkScowSchedulerApiVersion(client: SchedulerAdapterClient): Promise { + +// let scheduleApiVersion: ApiVersion | null; +// try { +// scheduleApiVersion = await asyncClientCall(client.version, "getVersion", {}); +// } catch (e) { +// const ex = e as ServiceError; +// const errors = parseErrorDetails(ex.metadata); +// // 如果接口不存在 +// if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED") { +// scheduleApiVersion = { major: 1, minor: 1, patch: 0 }; +// // 如果接口服务不存在 +// } else if ((e as any).code === status.UNIMPLEMENTED) { +// scheduleApiVersion = { major: 1, minor: 1, patch: 0 }; +// } else { +// scheduleApiVersion = null; +// logger.warn( +// "The scheduler API version can not be confirmed. Some functionalities may not operate as expected."); +// } +// } +// const scowSchedulerApiVersion = getCurrentScowSchedulerApiVersion(); + +// if (!scowSchedulerApiVersion && scheduleApiVersion) { +// logger.warn("The current scow scheduler API version can not be confirmed. Please ensure you are using " +// + "an API version that is compatible with the scheduler Api Version " +// + `${scheduleApiVersion.major}.${scheduleApiVersion.minor}.${scheduleApiVersion.patch}.`); +// } + +// if (scowSchedulerApiVersion && scheduleApiVersion) { +// const compareResult = compareSemVersion(scowSchedulerApiVersion, scheduleApiVersion); +// if (compareResult === 1) { +// logger.warn("The current scheduler API version is outdated. Some functionalities may not operate as expected. " +// + "Please upgrade to version " +// + +// `${scowSchedulerApiVersion.major}.${scowSchedulerApiVersion.minor}.${scowSchedulerApiVersion.patch} or later.`); +// } +// } + +// }; + + + + /** - * 判断当前集群下的调度器API版本是否已过时 + * 判断当前集群下的调度器API版本对比传入的接口是否已过时 * @param client - * @returns + * @param comparedVersion */ -export async function checkSchedulerApiVersion(client: SchedulerAdapterClient): Promise { +export async function checkSchedulerApiVersion(client: SchedulerAdapterClient, + comparedVersion: ApiVersion): Promise { let scheduleApiVersion: ApiVersion | null; try { @@ -54,20 +105,18 @@ export async function checkSchedulerApiVersion(client: SchedulerAdapterClient): "The scheduler API version can not be confirmed. Some functionalities may not operate as expected."); } } - const scowSchedulerApiVersion = getCurrentScowSchedulerApiVersion(); - - if (!scowSchedulerApiVersion && scheduleApiVersion) { - logger.warn("The current scow scheduler API version can not be confirmed. Please ensure you are using " - + "an API version that is compatible with the scheduler Api Version " - + `${scheduleApiVersion.major}.${scheduleApiVersion.minor}.${scheduleApiVersion.patch}.`); - } - if (scowSchedulerApiVersion && scheduleApiVersion) { - const compareResult = compareSemVersion(scowSchedulerApiVersion, scheduleApiVersion); + if (scheduleApiVersion) { + const compareResult = compareSemVersion(comparedVersion, scheduleApiVersion); if (compareResult === 1) { - logger.warn("The current scheduler API version is outdated. Some functionalities may not operate as expected. " - + "Please upgrade to version " - + `${scowSchedulerApiVersion.major}.${scowSchedulerApiVersion.minor}.${scowSchedulerApiVersion.patch} or later.`); + throw { + code: Status.FAILED_PRECONDITION, + message: "precondition failed", + details: "The method is not supported with the current scheduler adapter version. " + + "To use this method, the scheduler adapter must be upgraded to the version " + + `${comparedVersion.major}.${comparedVersion.minor}.${comparedVersion.patch}` + + "or higher.", + }; } } diff --git a/libs/utils/src/version.ts b/libs/utils/src/version.ts index d113a526b3..8a42bd062b 100644 --- a/libs/utils/src/version.ts +++ b/libs/utils/src/version.ts @@ -11,7 +11,7 @@ */ import { existsSync, readFileSync } from "fs"; -import { join } from "path"; +// import { join } from "path"; interface VersionJsonInfo { tag?: string; @@ -36,27 +36,27 @@ export type ApiVersion = { patch: number; } -// 获取当前调度器适配器接口仓库的API版本号 -export function getCurrentScowSchedulerApiVersion(): ApiVersion | null { +// // 获取当前调度器适配器接口仓库的API版本号 +// export function getCurrentScowSchedulerApiVersion(): ApiVersion | null { - const schedulerAdapterJsonFilePath = join(__dirname, "../../protos/scheduler-adapter/package.json"); - const packageJsonContent = JSON.parse(readFileSync(schedulerAdapterJsonFilePath, "utf-8")); +// const schedulerAdapterJsonFilePath = join(__dirname, "../../protos/scheduler-adapter/package.json"); +// const packageJsonContent = JSON.parse(readFileSync(schedulerAdapterJsonFilePath, "utf-8")); - const match = packageJsonContent.scripts.generate.match(/(?<=#tag=v)([\d.]+)/); +// const match = packageJsonContent.scripts.generate.match(/(?<=#tag=v)([\d.]+)/); - const version = match ? match[1] : undefined; +// const version = match ? match[1] : undefined; - if (version) { - const versionParts = version.split("."); - return { - major: parseInt(versionParts[0]), - minor: parseInt(versionParts[1]), - patch: parseInt(versionParts[2]), - }; - } else { - return null; - } -} +// if (version) { +// const versionParts = version.split("."); +// return { +// major: parseInt(versionParts[0]), +// minor: parseInt(versionParts[1]), +// patch: parseInt(versionParts[2]), +// }; +// } else { +// return null; +// } +// } /** * 比较Version1与Version2版本 From e4c1bdf5bd18a085c35cc49e92a4ee9f229a3dce Mon Sep 17 00:00:00 2001 From: picca Sun Date: Fri, 10 Nov 2023 03:43:00 +0000 Subject: [PATCH 13/13] fix --- .changeset/proud-seahorses-repeat.md | 13 ++- apps/portal-server/src/services/job.ts | 16 +--- apps/portal-server/src/utils/clusters.ts | 94 ++++++------------- .../src/pages/api/job/submitFileAsJob.ts | 4 + libs/utils/src/version.ts | 43 --------- 5 files changed, 44 insertions(+), 126 deletions(-) diff --git a/.changeset/proud-seahorses-repeat.md b/.changeset/proud-seahorses-repeat.md index 49cadf41e1..4cea5601e3 100644 --- a/.changeset/proud-seahorses-repeat.md +++ b/.changeset/proud-seahorses-repeat.md @@ -1,8 +1,11 @@ --- -"@scow/portal-server": patch -"@scow/test-adapter": patch -"@scow/portal-web": patch -"@scow/mis-web": patch +"@scow/scheduler-adapter-protos": minor +"@scow/lib-scheduler-adapter": minor +"@scow/portal-server": minor +"@scow/test-adapter": minor +"@scow/portal-web": minor +"@scow/mis-web": minor +"@scow/utils": minor --- -在门户系统的文件管理下，新增将文件直接作为作业文本提交调度器执行的功能 +在门户系统的文件管理下，新增将文件直接作为作业文本提交调度器执行的功能，如果调度器API版本低于此接口版本报错 diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index d0f58da6d6..de4c0a5f10 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -226,9 +226,10 @@ export const jobServiceServer = plugin((server) => { const client = getAdapterClient(cluster); if (!client) { throw clusterNotFound(cluster); } - const interfaceApiVersion: ApiVersion = { major: 1, minor: 2, patch: 0 }; - // 检验scow与调度器的API版本是否一致 - await checkSchedulerApiVersion(client, interfaceApiVersion); + // 当前接口要求的最低调度器接口版本 + const minRequiredApiVersion: ApiVersion = { major: 1, minor: 2, patch: 0 }; + // 检验调度器的API版本是否符合要求，不符合要求报错 + await checkSchedulerApiVersion(client, minRequiredApiVersion); const host = getClusterLoginNode(cluster); if (!host) { throw clusterNotFound(cluster); } @@ -278,15 +279,6 @@ export const jobServiceServer = plugin((server) => { message: "sbatch failed", details: e.details, }; - } else if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED") { - throw { - code: Status.FAILED_PRECONDITION, - message: "precondition failed", - details: "The method is not supported with the current scheduler adapter version. " - + "To use this method, the scheduler adapter must be upgraded to the version " - + `${interfaceApiVersion.major}.${interfaceApiVersion.minor}.${interfaceApiVersion.patch}` - + "or higher.", - }; } else { throw e; } diff --git a/apps/portal-server/src/utils/clusters.ts b/apps/portal-server/src/utils/clusters.ts index c8f56c83c7..ac94ae6c8f 100644 --- a/apps/portal-server/src/utils/clusters.ts +++ b/apps/portal-server/src/utils/clusters.ts @@ -15,11 +15,9 @@ import { ServiceError, status } from "@grpc/grpc-js"; import { Status } from "@grpc/grpc-js/build/src/constants"; import { getSchedulerAdapterClient, SchedulerAdapterClient } from "@scow/lib-scheduler-adapter"; import { parseErrorDetails } from "@scow/rich-error-model"; -import { ApiVersion, compareSemVersion } from "@scow/utils/build/version"; +import { ApiVersion } from "@scow/utils/build/version"; import { clusters } from "src/config/clusters"; -import { logger } from "./logger"; - const adapterClientForClusters = Object.entries(clusters).reduce((prev, [cluster, c]) => { const client = getSchedulerAdapterClient(c.adapterUrl); prev[cluster] = client; @@ -30,62 +28,13 @@ export const getAdapterClient = (cluster: string) => { return adapterClientForClusters[cluster]; }; - -// /** -// * 判断当前集群下的API版本与调度器API版本 -// * @param client -// * @returns -// */ -// export async function checkScowSchedulerApiVersion(client: SchedulerAdapterClient): Promise { - -// let scheduleApiVersion: ApiVersion | null; -// try { -// scheduleApiVersion = await asyncClientCall(client.version, "getVersion", {}); -// } catch (e) { -// const ex = e as ServiceError; -// const errors = parseErrorDetails(ex.metadata); -// // 如果接口不存在 -// if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED") { -// scheduleApiVersion = { major: 1, minor: 1, patch: 0 }; -// // 如果接口服务不存在 -// } else if ((e as any).code === status.UNIMPLEMENTED) { -// scheduleApiVersion = { major: 1, minor: 1, patch: 0 }; -// } else { -// scheduleApiVersion = null; -// logger.warn( -// "The scheduler API version can not be confirmed. Some functionalities may not operate as expected."); -// } -// } -// const scowSchedulerApiVersion = getCurrentScowSchedulerApiVersion(); - -// if (!scowSchedulerApiVersion && scheduleApiVersion) { -// logger.warn("The current scow scheduler API version can not be confirmed. Please ensure you are using " -// + "an API version that is compatible with the scheduler Api Version " -// + `${scheduleApiVersion.major}.${scheduleApiVersion.minor}.${scheduleApiVersion.patch}.`); -// } - -// if (scowSchedulerApiVersion && scheduleApiVersion) { -// const compareResult = compareSemVersion(scowSchedulerApiVersion, scheduleApiVersion); -// if (compareResult === 1) { -// logger.warn("The current scheduler API version is outdated. Some functionalities may not operate as expected. " -// + "Please upgrade to version " -// + -// `${scowSchedulerApiVersion.major}.${scowSchedulerApiVersion.minor}.${scowSchedulerApiVersion.patch} or later.`); -// } -// } - -// }; - - - - /** * 判断当前集群下的调度器API版本对比传入的接口是否已过时 * @param client - * @param comparedVersion + * @param minVersion */ export async function checkSchedulerApiVersion(client: SchedulerAdapterClient, - comparedVersion: ApiVersion): Promise { + minVersion: ApiVersion): Promise { let scheduleApiVersion: ApiVersion | null; try { @@ -93,28 +42,41 @@ export async function checkSchedulerApiVersion(client: SchedulerAdapterClient, } catch (e) { const ex = e as ServiceError; const errors = parseErrorDetails(ex.metadata); - // 如果接口不存在 - if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED") { - scheduleApiVersion = { major: 1, minor: 1, patch: 0 }; - // 如果接口服务不存在 - } else if ((e as any).code === status.UNIMPLEMENTED) { - scheduleApiVersion = { major: 1, minor: 1, patch: 0 }; + // 如果找不到获取版本号的接口，指定版本为接口存在前的最新版1.0.0 + if (((e as any).code === status.UNIMPLEMENTED) || + (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED")) { + scheduleApiVersion = { major: 1, minor: 0, patch: 0 }; } else { - scheduleApiVersion = null; - logger.warn( - "The scheduler API version can not be confirmed. Some functionalities may not operate as expected."); + throw { + code: Status.UNIMPLEMENTED, + message: "unimplemented", + details: "The scheduler API version can not be confirmed." + + "To use this method, the scheduler adapter must be upgraded to the version " + + `${minVersion.major}.${minVersion.minor}.${minVersion.patch} ` + + "or higher.", + }; } } if (scheduleApiVersion) { - const compareResult = compareSemVersion(comparedVersion, scheduleApiVersion); - if (compareResult === 1) { + + // 检查调度器接口版本是否大于等于最低要求版本 + let geMinVersion: boolean; + if (scheduleApiVersion.major !== minVersion.major) { + geMinVersion = (scheduleApiVersion.major > minVersion.major); + } else if (scheduleApiVersion.minor !== minVersion.minor) { + geMinVersion = (scheduleApiVersion.minor > minVersion.minor); + } else { + geMinVersion = true; + } + + if (!geMinVersion) { throw { code: Status.FAILED_PRECONDITION, message: "precondition failed", details: "The method is not supported with the current scheduler adapter version. " + "To use this method, the scheduler adapter must be upgraded to the version " - + `${comparedVersion.major}.${comparedVersion.minor}.${comparedVersion.patch}` + + `${minVersion.major}.${minVersion.minor}.${minVersion.patch} ` + "or higher.", }; } diff --git a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts index 49e95f4b06..57b553d1b4 100644 --- a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts +++ b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts @@ -48,6 +48,7 @@ export const SubmitFileAsJobSchema = typeboxRouteSchema({ code: Type.Union([ Type.Literal("SCHEDULER_FAILED"), Type.Literal("FAILED_PRECONDITION"), + Type.Literal("UNIMPLEMENTED"), ]), message: Type.String(), }), @@ -92,6 +93,9 @@ export default route(SubmitFileAsJobSchema, async (req, res) => { [status.FAILED_PRECONDITION]: () => ({ 500: { code: "FAILED_PRECONDITION" as const, message: "The method submitScriptAsJob is not supported with your current scheduler adapter version." } }), + [status.UNIMPLEMENTED]: () => ({ 500: { + code: "UNIMPLEMENTED" as const, + message: "The scheduler API version can not be confirmed." } }), [status.INVALID_ARGUMENT]: (err) => ({ 400: { code: "INVALID_ARGUMENT" as const, message: err.details } }), [status.PERMISSION_DENIED]: (err) => ({ 400: { code: "INVALID_PATH" as const, message: err.details } }), }, diff --git a/libs/utils/src/version.ts b/libs/utils/src/version.ts index 8a42bd062b..1738fe38d6 100644 --- a/libs/utils/src/version.ts +++ b/libs/utils/src/version.ts @@ -35,46 +35,3 @@ export type ApiVersion = { minor: number; patch: number; } - -// // 获取当前调度器适配器接口仓库的API版本号 -// export function getCurrentScowSchedulerApiVersion(): ApiVersion | null { - -// const schedulerAdapterJsonFilePath = join(__dirname, "../../protos/scheduler-adapter/package.json"); -// const packageJsonContent = JSON.parse(readFileSync(schedulerAdapterJsonFilePath, "utf-8")); - -// const match = packageJsonContent.scripts.generate.match(/(?<=#tag=v)([\d.]+)/); - -// const version = match ? match[1] : undefined; - -// if (version) { -// const versionParts = version.split("."); -// return { -// major: parseInt(versionParts[0]), -// minor: parseInt(versionParts[1]), -// patch: parseInt(versionParts[2]), -// }; -// } else { -// return null; -// } -// } - -/** - * 比较Version1与Version2版本 - * @param version1 - * @param version2 - * @returns - * 1： Version1高于Version2 - * -1：Version1低于Version2 - * 0: Version1与Version2版本相同 - */ -export function compareSemVersion(version1: ApiVersion, version2: ApiVersion): number { - if (version1.major !== version2.major) { - return version1.major > version2.major ? 1 : -1; - } else if (version1.minor !== version2.minor) { - return version1.minor > version2.minor ? 1 : -1; - } else if (version1.patch !== version2.patch) { - return version1.patch > version2.patch ? 1 : -1; - } else { - return 0; - } -};