From 135f2b1be375b9a7466dc70f2237cd373e133d61 Mon Sep 17 00:00:00 2001 From: Yixin Sun <43978285+piccaSun@users.noreply.github.com> Date: Sat, 11 Nov 2023 18:07:06 +0800 Subject: [PATCH] =?UTF-8?q?feat(portal):=20=E5=B0=86=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=9B=B4=E6=8E=A5=E6=8F=90=E4=BA=A4=E4=B8=BAsbatch=E5=91=BD?= =?UTF-8?q?=E4=BB=A4=20(#891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### 做了什么 **需求** 在文件管理中增加对所有文件对象增加”提交“按钮直接sbatch执行 暂时只考虑提交的文本作为作业直接执行 执行后也应该展示在正在运行的作业和已结束的作业中 提交文件大小限制为1M **实现** 此pr完成上述需求 增加文件是否为文本文件的判断 增加此操作在审计系统中的操作日志 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/44df586e-110b-42be-ac99-a224d69dd330) ![image](https://github.com/PKUHPC/SCOW/assets/43978285/c50febf4-dcee-48f5-8a9b-e4cd9ac69210) ![image](https://github.com/PKUHPC/SCOW/assets/43978285/63ceac2c-8398-4ced-b7ab-fe19c670b6a8) 提交失败时 ![fileSize-error](https://github.com/PKUHPC/SCOW/assets/43978285/aa6efe74-2955-430d-bd61-79086b5a82af) ![not-text](https://github.com/PKUHPC/SCOW/assets/43978285/b33edf48-aaa0-42ac-b62d-34dfe12118ce) ![sbatchfailed](https://github.com/PKUHPC/SCOW/assets/43978285/b562d79c-58b8-4419-9e98-0354e615237f) 操作日志追加 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/9a55c5c5-e5fd-4236-804c-9f16f56febd3) 调度器适配器接口仓库分支暂时变更为临时分支，测试结束后合并调度器适配器接口分支进主分支，修改复原SCOW中调度器适配器仓库 --- .changeset/cold-moons-bathe.md | 5 + .changeset/proud-seahorses-repeat.md | 11 ++ apps/mis-web/src/i18n/en.ts | 2 + apps/mis-web/src/i18n/zh_cn.ts | 2 + apps/mis-web/src/models/operationLog.ts | 5 + apps/portal-server/src/services/job.ts | 74 +++++++++++- apps/portal-server/src/utils/clusters.ts | 62 +++++++++- apps/portal-web/src/apis/api.mock.ts | 2 + apps/portal-web/src/apis/api.ts | 2 + apps/portal-web/src/i18n/en.ts | 5 + apps/portal-web/src/i18n/zh_cn.ts | 5 + apps/portal-web/src/models/operationLog.ts | 1 + .../filemanager/FileManager.tsx | 41 +++++++ .../src/pages/api/job/submitFileAsJob.ts | 109 ++++++++++++++++++ dev/test-adapter/src/app.ts | 2 + dev/test-adapter/src/services/job.ts | 4 + dev/test-adapter/src/services/version.ts | 25 ++++ libs/protos/scheduler-adapter/package.json | 2 +- libs/scheduler-adapter/src/client.ts | 3 + libs/utils/src/version.ts | 7 ++ protos/audit/operation_log.proto | 7 ++ protos/portal/job.proto | 15 +++ 22 files changed, 387 insertions(+), 4 deletions(-) create mode 100644 .changeset/cold-moons-bathe.md create mode 100644 .changeset/proud-seahorses-repeat.md create mode 100644 apps/portal-web/src/pages/api/job/submitFileAsJob.ts create mode 100644 dev/test-adapter/src/services/version.ts diff --git a/.changeset/cold-moons-bathe.md b/.changeset/cold-moons-bathe.md new file mode 100644 index 0000000000..c70fd7d1f4 --- /dev/null +++ b/.changeset/cold-moons-bathe.md @@ -0,0 +1,5 @@ +--- +"@scow/grpc-api": minor +--- + +新增 submitFileAsJob 接口，直接把文件作为作业提交调度器执行 diff --git a/.changeset/proud-seahorses-repeat.md b/.changeset/proud-seahorses-repeat.md new file mode 100644 index 0000000000..4cea5601e3 --- /dev/null +++ b/.changeset/proud-seahorses-repeat.md @@ -0,0 +1,11 @@ +--- +"@scow/scheduler-adapter-protos": minor +"@scow/lib-scheduler-adapter": minor +"@scow/portal-server": minor +"@scow/test-adapter": minor +"@scow/portal-web": minor +"@scow/mis-web": minor +"@scow/utils": minor +--- + +在门户系统的文件管理下，新增将文件直接作为作业文本提交调度器执行的功能，如果调度器API版本低于此接口版本报错 diff --git a/apps/mis-web/src/i18n/en.ts b/apps/mis-web/src/i18n/en.ts index b55cc477c8..9f5c7b444a 100644 --- a/apps/mis-web/src/i18n/en.ts +++ b/apps/mis-web/src/i18n/en.ts @@ -989,6 +989,7 @@ export default { setPlatformBilling: "Set Platform Job Billing", createTenant: "Create Tenant", tenantPay: "Tenant Recharge", + submitFileItemAsJob: "Script Submission", }, operationDetails: { login: "User Login", @@ -1041,6 +1042,7 @@ export default { createTenant: "Create tenant {}, administrator: {}", tenantPay: "Recharge tenant {} by {} yuan", setPlatformBilling: "Set platform billing item {} price to {} yuan", + submitFileItemAsJob: "Cluster: {}, Submit Script: {}", }, }, userRoles: { diff --git a/apps/mis-web/src/i18n/zh_cn.ts b/apps/mis-web/src/i18n/zh_cn.ts index 6f80275005..35005571fc 100644 --- a/apps/mis-web/src/i18n/zh_cn.ts +++ b/apps/mis-web/src/i18n/zh_cn.ts @@ -988,6 +988,7 @@ export default { setPlatformBilling: "设置平台作业计费", createTenant: "创建租户", tenantPay: "租户充值", + submitFileItemAsJob: "提交脚本", }, operationDetails: { login: "用户登录", @@ -1040,6 +1041,7 @@ export default { createTenant: "创建租户{}, 租户管理员为: {}", tenantPay: "为租户{}充值{}元", setPlatformBilling: "设置平台的计费项{}价格为{}元", + submitFileItemAsJob: "集群：{}，提交脚本：{}", }, }, userRoles: { diff --git a/apps/mis-web/src/models/operationLog.ts b/apps/mis-web/src/models/operationLog.ts index 4e2f94a0af..f631674414 100644 --- a/apps/mis-web/src/models/operationLog.ts +++ b/apps/mis-web/src/models/operationLog.ts @@ -76,6 +76,7 @@ export const OperationType: OperationTypeEnum = { setPlatformBilling: "setPlatformBilling", createTenant: "createTenant", tenantPay: "tenantPay", + submitFileItemAsJob: "submitFileItemAsJob", }; export const OperationLog = Type.Object({ @@ -163,6 +164,7 @@ export const getOperationTypeTexts = (t: OperationTextsTransType): { [key in Lib setPlatformBilling: t(pTypes("setPlatformBilling")), createTenant: t(pTypes("createTenant")), tenantPay: t(pTypes("tenantPay")), + submitFileItemAsJob: t(pTypes("submitFileItemAsJob")), }; }; @@ -186,6 +188,7 @@ export const OperationCodeMap: { [key in LibOperationType]: string } = { deleteDirectory: "010505", moveFileItem: "010506", copyFileItem: "010507", + submitFileItemAsJob: "010508", setJobTimeLimit: "010601", createUser: "020201", addUserToAccount: "020202", @@ -363,6 +366,8 @@ export const getOperationDetail = ( case "setPlatformBilling": return t(pDetails("setPlatformBilling"), [operationEvent[logEvent].path, nullableMoneyToString(operationEvent[logEvent].price)]); + case "submitFileItemAsJob": + return t(pDetails("submitFileItemAsJob"), [operationEvent[logEvent].clusterId, operationEvent[logEvent].path]); default: return "-"; } diff --git a/apps/portal-server/src/services/job.ts b/apps/portal-server/src/services/job.ts index 066b6856dc..de4c0a5f10 100644 --- a/apps/portal-server/src/services/job.ts +++ b/apps/portal-server/src/services/job.ts @@ -15,12 +15,13 @@ import { ServiceError } from "@ddadaal/tsgrpc-common"; import { plugin } from "@ddadaal/tsgrpc-server"; import { Status } from "@grpc/grpc-js/build/src/constants"; import { jobInfoToPortalJobInfo, jobInfoToRunningjob } from "@scow/lib-scheduler-adapter"; -import { createDirectoriesRecursively } from "@scow/lib-ssh"; +import { createDirectoriesRecursively, sftpReadFile, sftpStat } from "@scow/lib-ssh"; import { JobServiceServer, JobServiceService } from "@scow/protos/build/portal/job"; import { parseErrorDetails } from "@scow/rich-error-model"; +import { ApiVersion } from "@scow/utils/build/version"; import { getClusterOps } from "src/clusterops"; import { JobTemplate } from "src/clusterops/api/job"; -import { getAdapterClient } from "src/utils/clusters"; +import { checkSchedulerApiVersion, getAdapterClient } from "src/utils/clusters"; import { clusterNotFound } from "src/utils/errors"; import { getClusterLoginNode, sshConnect } from "src/utils/ssh"; @@ -218,6 +219,75 @@ export const jobServiceServer = plugin((server) => { return [{ jobId: reply.jobId }]; }, + + submitFileAsJob: async ({ request, logger }) => { + const { cluster, userId, filePath } = request; + + const client = getAdapterClient(cluster); + if (!client) { throw clusterNotFound(cluster); } + + // 当前接口要求的最低调度器接口版本 + const minRequiredApiVersion: ApiVersion = { major: 1, minor: 2, patch: 0 }; + // 检验调度器的API版本是否符合要求，不符合要求报错 + await checkSchedulerApiVersion(client, minRequiredApiVersion); + + const host = getClusterLoginNode(cluster); + if (!host) { throw clusterNotFound(cluster); } + + const script = await sshConnect(host, userId, logger, async (ssh) => { + + const sftp = await ssh.requestSFTP(); + + // 判断文件操作权限 + const stat = await sftpStat(sftp)(filePath).catch((e) => { + logger.error(e, "stat %s as %s failed", filePath, userId); + throw { + code: Status.PERMISSION_DENIED, message: `${filePath} is not accessible`, + }; + }); + // 文件SIZE大于1M不能提交sbatch执行 + if (stat.size / (1024 * 1024) > 1) { + throw { + code: Status.INVALID_ARGUMENT, message: `${filePath} is too large. Maximum file size is 1M`, + }; + } + + const isTextFile = await ssh.exec("file", [filePath]).then((res) => { + return res.match(/text/); + }); + // 文件不是文本文件不能提交Sbatch执行 + if (!isTextFile) { + throw { + code: Status.INVALID_ARGUMENT, message: `${filePath} is not a text file`, + }; + } + + return await sftpReadFile(sftp)(filePath) + .then((buffer) => { + return buffer.toString("utf-8"); + }); + }); + + const reply = await asyncClientCall(client.job, "submitScriptAsJob", { + userId, script, + }).catch((e) => { + const ex = e as ServiceError; + const errors = parseErrorDetails(ex.metadata); + if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "SBATCH_FAILED") { + throw { + code: Status.INTERNAL, + message: "sbatch failed", + details: e.details, + }; + } else { + throw e; + } + }); + + return [{ jobId: reply.jobId }]; + }, + + }); }); diff --git a/apps/portal-server/src/utils/clusters.ts b/apps/portal-server/src/utils/clusters.ts index 4ba6cdc40d..ac94ae6c8f 100644 --- a/apps/portal-server/src/utils/clusters.ts +++ b/apps/portal-server/src/utils/clusters.ts @@ -10,10 +10,14 @@ * See the Mulan PSL v2 for more details. */ +import { asyncClientCall } from "@ddadaal/tsgrpc-client"; +import { ServiceError, status } from "@grpc/grpc-js"; +import { Status } from "@grpc/grpc-js/build/src/constants"; import { getSchedulerAdapterClient, SchedulerAdapterClient } from "@scow/lib-scheduler-adapter"; +import { parseErrorDetails } from "@scow/rich-error-model"; +import { ApiVersion } from "@scow/utils/build/version"; import { clusters } from "src/config/clusters"; - const adapterClientForClusters = Object.entries(clusters).reduce((prev, [cluster, c]) => { const client = getSchedulerAdapterClient(c.adapterUrl); prev[cluster] = client; @@ -23,3 +27,59 @@ const adapterClientForClusters = Object.entries(clusters).reduce((prev, [cluster export const getAdapterClient = (cluster: string) => { return adapterClientForClusters[cluster]; }; + +/** + * 判断当前集群下的调度器API版本对比传入的接口是否已过时 + * @param client + * @param minVersion + */ +export async function checkSchedulerApiVersion(client: SchedulerAdapterClient, + minVersion: ApiVersion): Promise { + + let scheduleApiVersion: ApiVersion | null; + try { + scheduleApiVersion = await asyncClientCall(client.version, "getVersion", {}); + } catch (e) { + const ex = e as ServiceError; + const errors = parseErrorDetails(ex.metadata); + // 如果找不到获取版本号的接口，指定版本为接口存在前的最新版1.0.0 + if (((e as any).code === status.UNIMPLEMENTED) || + (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED")) { + scheduleApiVersion = { major: 1, minor: 0, patch: 0 }; + } else { + throw { + code: Status.UNIMPLEMENTED, + message: "unimplemented", + details: "The scheduler API version can not be confirmed." + + "To use this method, the scheduler adapter must be upgraded to the version " + + `${minVersion.major}.${minVersion.minor}.${minVersion.patch} ` + + "or higher.", + }; + } + } + + if (scheduleApiVersion) { + + // 检查调度器接口版本是否大于等于最低要求版本 + let geMinVersion: boolean; + if (scheduleApiVersion.major !== minVersion.major) { + geMinVersion = (scheduleApiVersion.major > minVersion.major); + } else if (scheduleApiVersion.minor !== minVersion.minor) { + geMinVersion = (scheduleApiVersion.minor > minVersion.minor); + } else { + geMinVersion = true; + } + + if (!geMinVersion) { + throw { + code: Status.FAILED_PRECONDITION, + message: "precondition failed", + details: "The method is not supported with the current scheduler adapter version. " + + "To use this method, the scheduler adapter must be upgraded to the version " + + `${minVersion.major}.${minVersion.minor}.${minVersion.patch} ` + + "or higher.", + }; + } + } + +}; diff --git a/apps/portal-web/src/apis/api.mock.ts b/apps/portal-web/src/apis/api.mock.ts index 3697b79077..e19efa5cc8 100644 --- a/apps/portal-web/src/apis/api.mock.ts +++ b/apps/portal-web/src/apis/api.mock.ts @@ -213,6 +213,8 @@ export const mockApi: MockApi = { submitJob: async () => ({ jobId: 10 }), + submitFileAsJob: async () => ({ jobId: 10 }), + getAppLastSubmission: async () => ({ lastSubmissionInfo: { userId: "test123", diff --git a/apps/portal-web/src/apis/api.ts b/apps/portal-web/src/apis/api.ts index e90f08f7b4..703f547528 100644 --- a/apps/portal-web/src/apis/api.ts +++ b/apps/portal-web/src/apis/api.ts @@ -54,6 +54,7 @@ import type { GetJobTemplateSchema } from "src/pages/api/job/getJobTemplate"; import type { GetRunningJobsSchema } from "src/pages/api/job/getRunningJobs"; import type { ListJobTemplatesSchema } from "src/pages/api/job/listJobTemplates"; import type { RenameJobTemplateSchema } from "src/pages/api/job/renameJobTemplate"; +import { SubmitFileAsJobSchema } from "src/pages/api/job/submitFileAsJob"; import type { SubmitJobSchema } from "src/pages/api/job/submitJob"; import type { ChangePasswordSchema } from "src/pages/api/profile/changePassword"; import type { CheckPasswordSchema } from "src/pages/api/profile/checkPassword"; @@ -97,6 +98,7 @@ export const api = { listJobTemplates: apiClient.fromTypeboxRoute("GET", "/api/job/listJobTemplates"), renameJobTemplate: apiClient.fromTypeboxRoute("POST", "/api/job/renameJobTemplate"), submitJob: apiClient.fromTypeboxRoute("POST", "/api/job/submitJob"), + submitFileAsJob: apiClient.fromTypeboxRoute("POST", "/api/job/submitFileAsJob"), changePassword: apiClient.fromTypeboxRoute("PATCH", "/api/profile/changePassword"), checkPassword: apiClient.fromTypeboxRoute("GET", "/api/profile/checkPassword"), startFileTransfer: apiClient.fromTypeboxRoute("PATCH", "/api/file/startFileTransfer"), diff --git a/apps/portal-web/src/i18n/en.ts b/apps/portal-web/src/i18n/en.ts index 0626bd6bac..b17be8d286 100644 --- a/apps/portal-web/src/i18n/en.ts +++ b/apps/portal-web/src/i18n/en.ts @@ -233,6 +233,11 @@ export default { deleteConfirmContent: "Confirm deletion of {}?", deleteConfirmOk: "Confirm", deleteSuccessMessage: "Deleted successfully", + submitConfirmTitle: "Submit Confirmation", + submitConfirmContent: "Confirm submission of {} to {}?", + submitConfirmOk: "Confirm", + submitSuccessMessage: "Submitted successfully! Your new job ID is: {}", + submitFailedMessage: "Submitted Failed", }, }, fileTable: { diff --git a/apps/portal-web/src/i18n/zh_cn.ts b/apps/portal-web/src/i18n/zh_cn.ts index 48bc10d0be..732da8733d 100644 --- a/apps/portal-web/src/i18n/zh_cn.ts +++ b/apps/portal-web/src/i18n/zh_cn.ts @@ -233,6 +233,11 @@ export default { deleteConfirmContent: "确认删除{}?", deleteConfirmOk: "确认", deleteSuccessMessage: "删除成功", + submitConfirmTitle: "确认提交", + submitConfirmContent: "确认提交{}至{}?", + submitConfirmOk: "确认", + submitSuccessMessage: "提交成功！您的新作业ID为：{}", + submitFailedMessage: "提交失败", }, }, fileTable: { diff --git a/apps/portal-web/src/models/operationLog.ts b/apps/portal-web/src/models/operationLog.ts index 01ee58379f..f86d2dfcdb 100644 --- a/apps/portal-web/src/models/operationLog.ts +++ b/apps/portal-web/src/models/operationLog.ts @@ -71,4 +71,5 @@ export const OperationType: OperationTypeEnum = { tenantPay: "tenantPay", blockAccount: "blockAccount", unblockAccount: "unblockAccount", + submitFileItemAsJob: "submitFileItemAsJob", }; diff --git a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx index d916c4756f..064b7a4ea0 100644 --- a/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/FileManager.tsx @@ -506,6 +506,47 @@ export const FileManager: React.FC = ({ cluster, path, urlPrefix }) => { > {t("button.deleteButton")} + { + i.type === "FILE" ? ( + { + const fullPath = join(path, i.name); + modal.confirm({ + title: t(p("tableInfo.submitConfirmTitle")), + content: t(p("tableInfo.submitConfirmContent"), + [i.name, getI18nConfigCurrentText(cluster.name, languageId)]), + okText: t(p("tableInfo.submitConfirmOk")), + onOk: async () => { + await api.submitFileAsJob({ + body: { + cluster: cluster.id, + filePath: fullPath, + }, + }) + .httpError(500, (e) => { + e.code === "SCHEDULER_FAILED" || e.code === "FAILED_PRECONDITION" ? modal.error({ + title: t(p("tableInfo.submitFailedMessage")), + content: e.message, + }) : (() => { throw e; })(); + }) + .httpError(400, (e) => { + e.code === "INVALID_ARGUMENT" || e.code === "INVALID_PATH" ? modal.error({ + title: t(p("tableInfo.submitFailedMessage")), + content: e.message, + }) : (() => { throw e; })(); + }) + .then((result) => { + message.success(t(p("tableInfo.submitSuccessMessage"), [result.jobId])); + resetSelectedAndOperation(); + reload(); + }); + }, + }); + }} + > + {t("button.submitButton")} + + ) : undefined + } )} /> diff --git a/apps/portal-web/src/pages/api/job/submitFileAsJob.ts b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts new file mode 100644 index 0000000000..57b553d1b4 --- /dev/null +++ b/apps/portal-web/src/pages/api/job/submitFileAsJob.ts @@ -0,0 +1,109 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { typeboxRouteSchema } from "@ddadaal/next-typed-api-routes-runtime"; +import { asyncUnaryCall } from "@ddadaal/tsgrpc-client"; +import { status } from "@grpc/grpc-js"; +import { JobServiceClient } from "@scow/protos/build/portal/job"; +import { Type } from "@sinclair/typebox"; +import { authenticate } from "src/auth/server"; +import { OperationResult, OperationType } from "src/models/operationLog"; +import { callLog } from "src/server/operationLog"; +import { getClient } from "src/utils/client"; +import { route } from "src/utils/route"; +import { handlegRPCError, parseIp } from "src/utils/server"; + +export const SubmitFileAsJobSchema = typeboxRouteSchema({ + method: "POST", + + body: Type.Object({ + cluster: Type.String(), + // 文件绝对路径 + filePath: Type.String(), + }), + + responses: { + 201: Type.Object({ + jobId: Type.Number(), + }), + + 400: Type.Object({ + code: Type.Union([ + Type.Literal("INVALID_ARGUMENT"), + Type.Literal("INVALID_PATH"), + ]), + message: Type.String(), + }), + + 500: Type.Object({ + code: Type.Union([ + Type.Literal("SCHEDULER_FAILED"), + Type.Literal("FAILED_PRECONDITION"), + Type.Literal("UNIMPLEMENTED"), + ]), + message: Type.String(), + }), + }, +}); + +const auth = authenticate(() => true); + +export default route(SubmitFileAsJobSchema, async (req, res) => { + + const info = await auth(req, res); + + if (!info) { return; } + + const { cluster, filePath } = req.body; + + const client = getClient(JobServiceClient); + + const logInfo = { + operatorUserId: info.identityId, + operatorIp: parseIp(req) ?? "", + operationTypePayload:{ + clusterId: cluster, path: filePath, + }, + }; + + return await asyncUnaryCall(client, "submitFileAsJob", { + cluster, userId: info.identityId, filePath + , + }) + .then(async ({ jobId }) => { + await callLog( + { ...logInfo, + operationTypeName: OperationType.submitFileItemAsJob, + operationTypePayload: { ... logInfo.operationTypePayload } }, + OperationResult.SUCCESS, + ); + return { 201: { jobId } } as const; + }) + .catch(handlegRPCError({ + [status.INTERNAL]: (err) => ({ 500: { code: "SCHEDULER_FAILED" as const, message: err.details } }), + [status.FAILED_PRECONDITION]: () => ({ 500: { + code: "FAILED_PRECONDITION" as const, + message: "The method submitScriptAsJob is not supported with your current scheduler adapter version." } }), + [status.UNIMPLEMENTED]: () => ({ 500: { + code: "UNIMPLEMENTED" as const, + message: "The scheduler API version can not be confirmed." } }), + [status.INVALID_ARGUMENT]: (err) => ({ 400: { code: "INVALID_ARGUMENT" as const, message: err.details } }), + [status.PERMISSION_DENIED]: (err) => ({ 400: { code: "INVALID_PATH" as const, message: err.details } }), + }, + async () => await callLog( + { ...logInfo, + operationTypeName: OperationType.submitFileItemAsJob, + operationTypePayload: { ... logInfo.operationTypePayload }, + }, + OperationResult.FAIL, + ))); +}); diff --git a/dev/test-adapter/src/app.ts b/dev/test-adapter/src/app.ts index 2b81ef8ec5..d7be925dfa 100644 --- a/dev/test-adapter/src/app.ts +++ b/dev/test-adapter/src/app.ts @@ -16,6 +16,7 @@ import { accountServiceServer } from "src/services/account"; import { configServiceServer } from "src/services/config"; import { jobServiceServer } from "src/services/job"; import { userServiceServer } from "src/services/user"; +import { versionServiceServer } from "src/services/version"; export async function createServer() { @@ -35,6 +36,7 @@ export async function createServer() { await server.register(userServiceServer); await server.register(jobServiceServer); await server.register(configServiceServer); + await server.register(versionServiceServer); return server; diff --git a/dev/test-adapter/src/services/job.ts b/dev/test-adapter/src/services/job.ts index ccdc9d31be..486d60f6f3 100644 --- a/dev/test-adapter/src/services/job.ts +++ b/dev/test-adapter/src/services/job.ts @@ -55,6 +55,10 @@ export const jobServiceServer = plugin((server) => { return [{ jobId: 1, generatedScript: "" }]; }, + submitScriptAsJob: async () => { + return [{ jobId: 1 }]; + }, + cancelJob: async () => { return [{}]; }, diff --git a/dev/test-adapter/src/services/version.ts b/dev/test-adapter/src/services/version.ts new file mode 100644 index 0000000000..e4931d8dc7 --- /dev/null +++ b/dev/test-adapter/src/services/version.ts @@ -0,0 +1,25 @@ +/** + * Copyright (c) 2022 Peking University and Peking University Institute for Computing and Digital Economy + * SCOW is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +import { plugin } from "@ddadaal/tsgrpc-server"; +import { VersionServiceServer, VersionServiceService } from "@scow/scheduler-adapter-protos/build/protos/version"; + +export const versionServiceServer = plugin((server) => { + server.addService(VersionServiceService, { + + getVersion: async () => { + return [{ major: 0, minor: 0, patch: 0 }]; + }, + + }); + +}); diff --git a/libs/protos/scheduler-adapter/package.json b/libs/protos/scheduler-adapter/package.json index 20cae6485e..9aa3b96b04 100644 --- a/libs/protos/scheduler-adapter/package.json +++ b/libs/protos/scheduler-adapter/package.json @@ -5,7 +5,7 @@ "main": "build/index.js", "private": true, "scripts": { - "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git#tag=v1.0.1", + "generate": "rimraf generated && buf generate --template buf.gen.yaml https://github.com/PKUHPC/scow-scheduler-adapter-interface.git#tag=v1.2.0", "build": "rimraf build && tsc" }, "files": [ diff --git a/libs/scheduler-adapter/src/client.ts b/libs/scheduler-adapter/src/client.ts index dd9587497c..0baafc76dd 100644 --- a/libs/scheduler-adapter/src/client.ts +++ b/libs/scheduler-adapter/src/client.ts @@ -15,6 +15,7 @@ import { AccountServiceClient } from "@scow/scheduler-adapter-protos/build/proto import { ConfigServiceClient } from "@scow/scheduler-adapter-protos/build/protos/config"; import { JobServiceClient } from "@scow/scheduler-adapter-protos/build/protos/job"; import { UserServiceClient } from "@scow/scheduler-adapter-protos/build/protos/user"; +import { VersionServiceClient } from "@scow/scheduler-adapter-protos/build/protos/version"; type ClientConstructor = new (address: string, credentials: ChannelCredentials) => TClient; @@ -24,6 +25,7 @@ export interface SchedulerAdapterClient { user: UserServiceClient; job: JobServiceClient; config: ConfigServiceClient; + version: VersionServiceClient; } export function getClient( @@ -41,5 +43,6 @@ export const getSchedulerAdapterClient = (address: string) => { user: getClient(address, UserServiceClient), job: getClient(address, JobServiceClient), config: getClient(address, ConfigServiceClient), + version: getClient(address, VersionServiceClient), }; }; diff --git a/libs/utils/src/version.ts b/libs/utils/src/version.ts index 46ff848ee8..1738fe38d6 100644 --- a/libs/utils/src/version.ts +++ b/libs/utils/src/version.ts @@ -11,6 +11,7 @@ */ import { existsSync, readFileSync } from "fs"; +// import { join } from "path"; interface VersionJsonInfo { tag?: string; @@ -28,3 +29,9 @@ export function readVersionFile(versionJsonFileName = "version.json") { return jsonInfo; } +// SemVer类型version +export type ApiVersion = { + major: number; + minor: number; + patch: number; +} diff --git a/protos/audit/operation_log.proto b/protos/audit/operation_log.proto index 9094c69f95..b63e3eb0aa 100644 --- a/protos/audit/operation_log.proto +++ b/protos/audit/operation_log.proto @@ -111,6 +111,11 @@ message CopyFileItem { string to_path = 3; } +message SubmitFileItemAsJob { + string cluster_id = 1; + string path = 2; +} + message SetJobTimeLimit { string account_name = 1; uint32 job_id = 2; @@ -331,6 +336,7 @@ message CreateOperationLogRequest { SetPlatformBilling set_platform_billing = 50; CreateTenant create_tenant = 51; TenantPay tenant_pay = 52; + SubmitFileItemAsJob submit_file_item_as_job = 53; } } @@ -391,6 +397,7 @@ message OperationLog { SetPlatformBilling set_platform_billing = 52; CreateTenant create_tenant = 53; TenantPay tenant_pay = 54; + SubmitFileItemAsJob submit_file_item_as_job = 55; } } diff --git a/protos/portal/job.proto b/protos/portal/job.proto index 419381d19b..16fb669625 100644 --- a/protos/portal/job.proto +++ b/protos/portal/job.proto @@ -159,6 +159,19 @@ message SubmitJobResponse { uint32 job_id = 1; } +// filePath: file's absolute path +message SubmitFileAsJobRequest { + string cluster = 1; + string user_id = 2; + string file_path = 3; +} + +// NOT_FOUND: cluster is not found +// INTERNAL: error raised from scheduler. details has the error message +message SubmitFileAsJobResponse { + uint32 job_id = 1; +} + service JobService { rpc CancelJob(CancelJobRequest) returns (CancelJobResponse); @@ -177,4 +190,6 @@ service JobService { rpc RenameJobTemplate(RenameJobTemplateRequest) returns (RenameJobTemplateResponse); rpc SubmitJob(SubmitJobRequest) returns (SubmitJobResponse); + + rpc SubmitFileAsJob(SubmitFileAsJobRequest) returns (SubmitFileAsJobResponse); }