feat(portal): 将文件直接提交为sbatch命令

.changeset/cold-moons-bathe.md

@@ -0,0 +1,5 @@
+---
+"@scow/grpc-api": minor
+---
+
+新增 submitFileAsJob 接口，直接把文件作为作业提交调度器执行

.changeset/proud-seahorses-repeat.md

@@ -0,0 +1,11 @@
+---
+"@scow/scheduler-adapter-protos": minor
+"@scow/lib-scheduler-adapter": minor
+"@scow/portal-server": minor
+"@scow/test-adapter": minor
+"@scow/portal-web": minor
+"@scow/mis-web": minor
+"@scow/utils": minor
+---
+
+在门户系统的文件管理下，新增将文件直接作为作业文本提交调度器执行的功能，如果调度器API版本低于此接口版本报错

apps/mis-web/src/i18n/en.ts

@@ -986,6 +986,7 @@ export default {
       setPlatformBilling: "Set Platform Job Billing",
       createTenant: "Create Tenant",
       tenantPay: "Tenant Recharge",
+      submitFileItemAsJob: "Script Submission",
     },
     operationDetails: {
       login: "User Login",
@@ -1038,6 +1039,7 @@ export default {
       createTenant: "Create tenant {}, administrator: {}",
       tenantPay: "Recharge tenant {} by {} yuan",
       setPlatformBilling: "Set platform billing item {} price to {} yuan",
+      submitFileItemAsJob: "Cluster: {}, Submit Script: {}",
     },
   },
   userRoles: {

apps/mis-web/src/i18n/zh_cn.ts

@@ -985,6 +985,7 @@ export default {
       setPlatformBilling: "设置平台作业计费",
       createTenant: "创建租户",
       tenantPay: "租户充值",
+      submitFileItemAsJob: "提交脚本",
     },
     operationDetails: {
       login: "用户登录",
@@ -1037,6 +1038,7 @@ export default {
       createTenant: "创建租户{}, 租户管理员为: {}",
       tenantPay: "为租户{}充值{}元",
       setPlatformBilling: "设置平台的计费项{}价格为{}元",
+      submitFileItemAsJob: "集群：{}，提交脚本：{}",
     },
   },
   userRoles: {

apps/mis-web/src/models/operationLog.ts

@@ -76,6 +76,7 @@ export const OperationType: OperationTypeEnum = {
   setPlatformBilling: "setPlatformBilling",
   createTenant: "createTenant",
   tenantPay: "tenantPay",
+  submitFileItemAsJob: "submitFileItemAsJob",
 };
 
 export const OperationLog = Type.Object({
@@ -163,6 +164,7 @@ export const getOperationTypeTexts = (t: OperationTextsTransType): { [key in Lib
     setPlatformBilling: t(pTypes("setPlatformBilling")),
     createTenant: t(pTypes("createTenant")),
     tenantPay: t(pTypes("tenantPay")),
+    submitFileItemAsJob: t(pTypes("submitFileItemAsJob")),
   };
 
 };
@@ -186,6 +188,7 @@ export const OperationCodeMap: { [key in LibOperationType]: string } = {
   deleteDirectory: "010505",
   moveFileItem: "010506",
   copyFileItem: "010507",
+  submitFileItemAsJob: "010508",
   setJobTimeLimit: "010601",
   createUser: "020201",
   addUserToAccount: "020202",
@@ -363,6 +366,8 @@ export const getOperationDetail = (
     case "setPlatformBilling":
       return t(pDetails("setPlatformBilling"),
         [operationEvent[logEvent].path, nullableMoneyToString(operationEvent[logEvent].price)]);
+    case "submitFileItemAsJob":
+      return t(pDetails("submitFileItemAsJob"), [operationEvent[logEvent].clusterId, operationEvent[logEvent].path]);
     default:
       return "-";
     }

apps/portal-server/src/services/job.ts

@@ -15,12 +15,13 @@ import { ServiceError } from "@ddadaal/tsgrpc-common";
 import { plugin } from "@ddadaal/tsgrpc-server";
 import { Status } from "@grpc/grpc-js/build/src/constants";
 import { jobInfoToPortalJobInfo, jobInfoToRunningjob } from "@scow/lib-scheduler-adapter";
-import { createDirectoriesRecursively } from "@scow/lib-ssh";
+import { createDirectoriesRecursively, sftpReadFile, sftpStat } from "@scow/lib-ssh";
 import { JobServiceServer, JobServiceService } from "@scow/protos/build/portal/job";
 import { parseErrorDetails } from "@scow/rich-error-model";
+import { ApiVersion } from "@scow/utils/build/version";
 import { getClusterOps } from "src/clusterops";
 import { JobTemplate } from "src/clusterops/api/job";
-import { getAdapterClient } from "src/utils/clusters";
+import { checkSchedulerApiVersion, getAdapterClient } from "src/utils/clusters";
 import { clusterNotFound } from "src/utils/errors";
 import { getClusterLoginNode, sshConnect } from "src/utils/ssh";
 
@@ -218,6 +219,75 @@ export const jobServiceServer = plugin((server) => {
       return [{ jobId: reply.jobId }];
     },
 
+
+    submitFileAsJob: async ({ request, logger }) => {
+      const { cluster, userId, filePath } = request;
+
+      const client = getAdapterClient(cluster);
+      if (!client) { throw clusterNotFound(cluster); }
+
+      // 当前接口要求的最低调度器接口版本
+      const minRequiredApiVersion: ApiVersion = { major: 1, minor: 2, patch: 0 };
+      // 检验调度器的API版本是否符合要求，不符合要求报错
+      await checkSchedulerApiVersion(client, minRequiredApiVersion);
+
+      const host = getClusterLoginNode(cluster);
+      if (!host) { throw clusterNotFound(cluster); }
+
+      const script = await sshConnect(host, userId, logger, async (ssh) => {
+
+        const sftp = await ssh.requestSFTP();
+
+        // 判断文件操作权限
+        const stat = await sftpStat(sftp)(filePath).catch((e) => {
+          logger.error(e, "stat %s as %s failed", filePath, userId);
+          throw <ServiceError> {
+            code: Status.PERMISSION_DENIED, message: `${filePath} is not accessible`,
+          };
+        });
+        // 文件SIZE大于1M不能提交sbatch执行
+        if (stat.size / (1024 * 1024) > 1) {
+          throw <ServiceError> {
+            code: Status.INVALID_ARGUMENT, message: `${filePath} is too large. Maximum file size is 1M`,
+          };
+        }
+
+        const isTextFile = await ssh.exec("file", [filePath]).then((res) => {
+          return res.match(/text/);
+        });
+        // 文件不是文本文件不能提交Sbatch执行
+        if (!isTextFile) {
+          throw <ServiceError> {
+            code: Status.INVALID_ARGUMENT, message: `${filePath} is not a text file`,
+          };
+        }
+
+        return await sftpReadFile(sftp)(filePath)
+          .then((buffer) => {
+            return buffer.toString("utf-8");
+          });
+      });
+
+      const reply = await asyncClientCall(client.job, "submitScriptAsJob", {
+        userId, script,
+      }).catch((e) => {
+        const ex = e as ServiceError;
+        const errors = parseErrorDetails(ex.metadata);
+        if (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "SBATCH_FAILED") {
+          throw <ServiceError> {
+            code: Status.INTERNAL,
+            message: "sbatch failed",
+            details: e.details,
+          };
+        } else {
+          throw e;
+        }
+      });
+
+      return [{ jobId: reply.jobId }];
+    },
+
+
   });
 
 });

apps/portal-server/src/utils/clusters.ts

@@ -10,10 +10,14 @@
  * See the Mulan PSL v2 for more details.
  */
 
+import { asyncClientCall } from "@ddadaal/tsgrpc-client";
+import { ServiceError, status } from "@grpc/grpc-js";
+import { Status } from "@grpc/grpc-js/build/src/constants";
 import { getSchedulerAdapterClient, SchedulerAdapterClient } from "@scow/lib-scheduler-adapter";
+import { parseErrorDetails } from "@scow/rich-error-model";
+import { ApiVersion } from "@scow/utils/build/version";
 import { clusters } from "src/config/clusters";
 
-
 const adapterClientForClusters = Object.entries(clusters).reduce((prev, [cluster, c]) => {
   const client = getSchedulerAdapterClient(c.adapterUrl);
   prev[cluster] = client;
@@ -23,3 +27,59 @@
 export const getAdapterClient = (cluster: string) => {
   return adapterClientForClusters[cluster];
 };
+
+/**
+ * 判断当前集群下的调度器API版本对比传入的接口是否已过时
+ * @param client
+ * @param minVersion
+ */
+export async function checkSchedulerApiVersion(client: SchedulerAdapterClient,
+  minVersion: ApiVersion): Promise<void> {
+
+  let scheduleApiVersion: ApiVersion | null;
+  try {
+    scheduleApiVersion = await asyncClientCall(client.version, "getVersion", {});
+  } catch (e) {
+    const ex = e as ServiceError;
+    const errors = parseErrorDetails(ex.metadata);
+    // 如果找不到获取版本号的接口，指定版本为接口存在前的最新版1.0.0
+    if (((e as any).code === status.UNIMPLEMENTED) ||
+    (errors[0] && errors[0].$type === "google.rpc.ErrorInfo" && errors[0].reason === "UNIMPLEMENTED")) {
+      scheduleApiVersion = { major: 1, minor: 0, patch: 0 };
+    } else {
+      throw <ServiceError> {
+        code: Status.UNIMPLEMENTED,
+        message: "unimplemented",
+        details: "The scheduler API version can not be confirmed."
+          + "To use this method, the scheduler adapter must be upgraded to the version "
+          + `${minVersion.major}.${minVersion.minor}.${minVersion.patch} `
+          + "or higher.",
+      };
+    }
+  }
+
+  if (scheduleApiVersion) {
+
+    // 检查调度器接口版本是否大于等于最低要求版本
+    let geMinVersion: boolean;
+    if (scheduleApiVersion.major !== minVersion.major) {
+      geMinVersion = (scheduleApiVersion.major > minVersion.major);
+    } else if (scheduleApiVersion.minor !== minVersion.minor) {
+      geMinVersion = (scheduleApiVersion.minor > minVersion.minor);
+    } else {
+      geMinVersion = true;
+    }
+
+    if (!geMinVersion) {
+      throw <ServiceError> {
+        code: Status.FAILED_PRECONDITION,
+        message: "precondition failed",
+        details: "The method is not supported with the current scheduler adapter version. "
+          + "To use this method, the scheduler adapter must be upgraded to the version "
+          + `${minVersion.major}.${minVersion.minor}.${minVersion.patch} `
+          + "or higher.",
+      };
+    }
+  }
+
+};

apps/portal-web/src/apis/api.mock.ts

@@ -213,6 +213,8 @@ export const mockApi: MockApi<typeof api> = {
 
   submitJob: async () => ({ jobId: 10 }),
 
+  submitFileAsJob: async () => ({ jobId: 10 }),
+
   getAppLastSubmission: async () => ({
     lastSubmissionInfo: {
       userId: "test123",

apps/portal-web/src/apis/api.ts

@@ -54,6 +54,7 @@ import type { GetJobTemplateSchema } from "src/pages/api/job/getJobTemplate";
 import type { GetRunningJobsSchema } from "src/pages/api/job/getRunningJobs";
 import type { ListJobTemplatesSchema } from "src/pages/api/job/listJobTemplates";
 import type { RenameJobTemplateSchema } from "src/pages/api/job/renameJobTemplate";
+import { SubmitFileAsJobSchema } from "src/pages/api/job/submitFileAsJob";
 import type { SubmitJobSchema } from "src/pages/api/job/submitJob";
 import type { ChangePasswordSchema } from "src/pages/api/profile/changePassword";
 import type { CheckPasswordSchema } from "src/pages/api/profile/checkPassword";
@@ -97,6 +98,7 @@ export const api = {
   listJobTemplates: apiClient.fromTypeboxRoute<typeof ListJobTemplatesSchema>("GET", "/api/job/listJobTemplates"),
   renameJobTemplate: apiClient.fromTypeboxRoute<typeof RenameJobTemplateSchema>("POST", "/api/job/renameJobTemplate"),
   submitJob: apiClient.fromTypeboxRoute<typeof SubmitJobSchema>("POST", "/api/job/submitJob"),
+  submitFileAsJob: apiClient.fromTypeboxRoute<typeof SubmitFileAsJobSchema>("POST", "/api/job/submitFileAsJob"),
   changePassword: apiClient.fromTypeboxRoute<typeof ChangePasswordSchema>("PATCH", "/api/profile/changePassword"),
   checkPassword: apiClient.fromTypeboxRoute<typeof CheckPasswordSchema>("GET", "/api/profile/checkPassword"),
   startFileTransfer: apiClient.fromTypeboxRoute<typeof StartFileTransferSchema>("PATCH", "/api/file/startFileTransfer"),

apps/portal-web/src/i18n/en.ts

@@ -234,6 +234,11 @@ export default {
           deleteConfirmContent: "Confirm deletion of {}?",
           deleteConfirmOk: "Confirm",
           deleteSuccessMessage: "Deleted successfully",
+          submitConfirmTitle: "Submit Confirmation",
+          submitConfirmContent: "Confirm submission of {} to {}?",
+          submitConfirmOk: "Confirm",
+          submitSuccessMessage: "Submitted successfully! Your new job ID is: {}",
+          submitFailedMessage: "Submitted Failed",
         },
       },
       fileTable: {

apps/portal-web/src/i18n/zh_cn.ts

@@ -234,6 +234,11 @@ export default {
           deleteConfirmContent: "确认删除{}?",
           deleteConfirmOk: "确认",
           deleteSuccessMessage: "删除成功",
+          submitConfirmTitle: "确认提交",
+          submitConfirmContent: "确认提交{}至{}?",
+          submitConfirmOk: "确认",
+          submitSuccessMessage: "提交成功！您的新作业ID为：{}",
+          submitFailedMessage: "提交失败",
         },
       },
       fileTable: {

apps/portal-web/src/models/operationLog.ts

@@ -71,4 +71,5 @@ export const OperationType: OperationTypeEnum = {
   tenantPay: "tenantPay",
   blockAccount: "blockAccount",
   unblockAccount: "unblockAccount",
+  submitFileItemAsJob: "submitFileItemAsJob",
 };

apps/portal-web/src/pageComponents/filemanager/FileManager.tsx

@@ -506,6 +506,47 @@ export const FileManager: React.FC<Props> = ({ cluster, path, urlPrefix }) => {
             >
               {t("button.deleteButton")}
             </a>
+            {
+              i.type === "FILE" ? (
+                <a onClick={() => {
+                  const fullPath = join(path, i.name);
+                  modal.confirm({
+                    title: t(p("tableInfo.submitConfirmTitle")),
+                    content: t(p("tableInfo.submitConfirmContent"),
+                      [i.name, getI18nConfigCurrentText(cluster.name, languageId)]),
+                    okText: t(p("tableInfo.submitConfirmOk")),
+                    onOk: async () => {
+                      await api.submitFileAsJob({
+                        body: {
+                          cluster: cluster.id,
+                          filePath: fullPath,
+                        },
+                      })
+                        .httpError(500, (e) => {
+                          e.code === "SCHEDULER_FAILED" || e.code === "FAILED_PRECONDITION" ? modal.error({
+                            title: t(p("tableInfo.submitFailedMessage")),
+                            content: e.message,
+                          }) : (() => { throw e; })();
+                        })
+                        .httpError(400, (e) => {
+                          e.code === "INVALID_ARGUMENT" || e.code === "INVALID_PATH" ? modal.error({
+                            title: t(p("tableInfo.submitFailedMessage")),
+                            content: e.message,
+                          }) : (() => { throw e; })();
+                        })
+                        .then((result) => {
+                          message.success(t(p("tableInfo.submitSuccessMessage"), [result.jobId]));
+                          resetSelectedAndOperation();
+                          reload();
+                        });
+                    },
+                  });
+                }}
+                >
+                  {t("button.submitButton")}
+                </a>
+              ) : undefined
+            }
           </Space>
         )}
       />