diff --git a/app/app.less b/app/app.less index 40982178..09b557b7 100644 --- a/app/app.less +++ b/app/app.less @@ -56,6 +56,7 @@ border: none; white-space: nowrap; color: @darkBlue; + &.ant-radio-button-wrapper-checked { color: #fff; } diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index f77ebfe4..d702932f 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -247,9 +247,10 @@ "createTask": "New Import", "uploadTemp": "Import Template", "downloadConfig": "Download Config", + "downloadLog": "Download Log", "viewLogs": "View Logs", "details": "Details", - "lines": "Lines", + "task": "import task", "taskList": "Task List", "taskName": "Task Name", "vertices": "Map Vertices", @@ -332,10 +333,16 @@ "importCompleted": "Import completed", "importStopped": "Import stopped", "importFailed": "Failed", - "notImported": "{total} lines not imported", - "readFailed": "{total} lines read failed", + "notImported": "{total} not imported", + "readFailed": "{total} read failed", "selectFile": "Select bind source file", - "addTag": "Add Tag" + "addTag": "Add Tag", + "config": "Task Config", + "parseFailed": "File parsing failed", + "uploadTemplate": "Click or drag the yaml configuration file to this area to upload", + "uploadTemplateTip": "Please keep only the file name (retain the file extension) for all file paths in the template, such as logPath: config.csv", + "fileUploadRequired": "Make sure all csv data files are uploaded before uploading the configuration", + "reUpload": "Re-upload" }, "schema": { "spaceList": "Graph Space List", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index aef52601..7dc9f84f 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -247,9 +247,10 @@ "createTask": "创建导入任务", "uploadTemp": "导入模板", "downloadConfig": "下载配置文件", + "downloadLog": "下载日志", "viewLogs": "查看日志", "details": "详情", - "lines": "行", + "task": "导入任务", "taskList": "任务列表", "taskName": "任务名称", "vertices": "关联点", @@ -328,10 +329,16 @@ "importCompleted": "导入完成", "importStopped": "导入中止", "importFailed": "导入失败", - "notImported": "{total}行未导入", - "readFailed": "{total}行读取失败", + "notImported": "{total}未导入", + "readFailed": "{total}读取失败", "selectFile": "选择绑定文件", - "addTag": "添加 Tag" + "addTag": "添加 Tag", + "config": "任务配置", + "parseFailed": "文件解析失败", + "uploadTemplate": "点击或拖动yaml配置文件到该区域上传", + "uploadTemplateTip": "模板中所有文件路径请仅保留文件名(保留文件扩展名），比如 logPath: config.csv", + "fileUploadRequired": "上传配置前请确保所有 csv 数据文件已上传", + "reUpload": "重新上传" }, "schema": { "spaceList": "图空间列表", diff --git a/app/config/service.ts b/app/config/service.ts index efb50873..63ec07a1 100644 --- a/app/config/service.ts +++ b/app/config/service.ts @@ -13,6 +13,7 @@ const importData = post('/api-nebula/task/import'); const handleImportAction = post('/api-nebula/task/import/action'); const getLog = get('/api/import/log'); +const getErrLog = get('/api/import/err_log'); const finishImport = post('/api/import/finish'); const getImportWokingDir = get('/api/import/working_dir'); @@ -31,6 +32,15 @@ const uploadFiles = (params?, config?) => 'Content-Type': 'multipart/form-data', }, }); + + +const getTaskLogs = (params?, config?) => { + const { id, ...others } = params; + return get(`/api/import/task_log_paths/${id}`)(others, config); +}; + +const getTaskConfigUrl = (id: number) => `/api-nebula/task/import/config/${id}`; +const getTaskLogUrl = (path: string) => `/api-nebula/task/import/log?pathName=${encodeURI(path)}`; export default { execNGQL, batchExecNGQL, @@ -40,10 +50,14 @@ export default { finishImport, handleImportAction, getLog, + getErrLog, getImportWokingDir, getUploadDir, getTaskDir, deteleFile, getFiles, uploadFiles, + getTaskConfigUrl, + getTaskLogs, + getTaskLogUrl }; diff --git a/app/interfaces/import.ts b/app/interfaces/import.ts index 30e4f69e..61391df1 100644 --- a/app/interfaces/import.ts +++ b/app/interfaces/import.ts @@ -8,8 +8,11 @@ export enum ITaskStatus { } export interface ITaskStats { - totalLine: number; - totalCount: number; + totalBatches: number; + totalBytes: number; + totalImportedBytes: number; + totalLatency: number; + totalReqTime: number; numFailed: number; numReadFailed: number; } @@ -23,7 +26,7 @@ export interface ITaskItem { user: string; taskStatus: ITaskStatus; taskMessage: string; - statsQuery: ITaskStats; + stats: ITaskStats; } export interface IVerticesConfig { diff --git a/app/pages/Console/OutputBox/index.tsx b/app/pages/Console/OutputBox/index.tsx index ee48d2b9..f7e65d28 100644 --- a/app/pages/Console/OutputBox/index.tsx +++ b/app/pages/Console/OutputBox/index.tsx @@ -169,9 +169,7 @@ const OutputBox = (props: IProps) => { const link = document.createElement('a'); link.href = url; link.download = `result.csv`; - document.body.appendChild(link); link.click(); - document.body.removeChild(link); }; const handleExplore = () => { diff --git a/app/pages/Console/index.tsx b/app/pages/Console/index.tsx index 6b351bab..9ef50897 100644 --- a/app/pages/Console/index.tsx +++ b/app/pages/Console/index.tsx @@ -2,7 +2,7 @@ import { Button, Select, Tooltip, message } from 'antd'; import React, { useEffect, useRef, useState } from 'react'; import intl from 'react-intl-universal'; import { observer } from 'mobx-react-lite'; -import { trackPageView, trackEvent } from '@app/utils/stat'; +import { trackEvent, trackPageView } from '@app/utils/stat'; import { useStore } from '@app/stores'; import Instruction from '@app/components/Instruction'; import Icon from '@app/components/Icon'; diff --git a/app/pages/Import/TaskCreate/SchemaConfig/TagConfig/index.tsx b/app/pages/Import/TaskCreate/SchemaConfig/TagConfig/index.tsx index f0d4c7d2..16eade44 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/TagConfig/index.tsx +++ b/app/pages/Import/TaskCreate/SchemaConfig/TagConfig/index.tsx @@ -17,11 +17,11 @@ interface IProps { const VerticesConfig = (props: IProps) => { const { tag, tagIndex, configIndex, file } = props; const { dataImport, schema } = useStore(); - const { asyncUpdateTagConfig, updateTagPropMapping } = dataImport; + const { updateTagConfig, updateTagPropMapping } = dataImport; const { tags } = schema; const handleTagChange = (configIndex: number, tagIndex: number, value: string) => { - asyncUpdateTagConfig({ configIndex, tagIndex, tag: value }); + updateTagConfig({ configIndex, tagIndex, tag: value }); }; const handlePropChange = (index, field, value) => { diff --git a/app/pages/Import/TaskCreate/index.tsx b/app/pages/Import/TaskCreate/index.tsx index 38767aee..a185847d 100644 --- a/app/pages/Import/TaskCreate/index.tsx +++ b/app/pages/Import/TaskCreate/index.tsx @@ -24,7 +24,7 @@ const formItemLayout = { }; const TaskCreate = () => { const { dataImport, schema, global } = useStore(); - const { taskDir, asyncGetTaskDir, basicConfig, verticesConfig, edgesConfig, updateBasicConfig, importTask } = dataImport; + const { taskDir, getTaskDir, basicConfig, verticesConfig, edgesConfig, updateBasicConfig, importTask } = dataImport; const { spaces, spaceVidType, getSpaces, updateSpaceInfo, currentSpace } = schema; const { host, username } = global; const { batchSize } = basicConfig; @@ -125,7 +125,7 @@ const TaskCreate = () => { }; useEffect(() => { - asyncGetTaskDir(); + getTaskDir(); getSpaces(); if(currentSpace) { updateSpaceInfo(currentSpace); diff --git a/app/pages/Import/TaskList/TaskItem/LogModal/index.less b/app/pages/Import/TaskList/TaskItem/LogModal/index.less new file mode 100644 index 00000000..4a3d09f7 --- /dev/null +++ b/app/pages/Import/TaskList/TaskItem/LogModal/index.less @@ -0,0 +1,62 @@ +@import '~@app/common.less'; +.log-modal { + height: 100vh; + .ant-modal { + height: 80%; + .ant-modal-content { + height: 100%; + } + } + .import-modal-title { + display: flex; + align-items: center; + } + .ant-modal-header { + border-bottom: none; + padding-right: 80px; + padding-top: 15px; + .ant-modal-title { + display: flex; + align-items: center; + justify-content: space-between; + } + .ant-modal-close { + top: 5px; + } + } + .ant-modal-body { + display: flex; + height: 91%; + } + .log-container { + width: 100%; + height: 100%; + overflow: auto; + padding: 10px 20px 120px; + font-size: 18px; + text-align: left; + background: #333; + color: #fff; + word-break: break-all; + } +} +.log-tab { + max-height: 65vh; + .ant-tabs-nav { + width: 200px; + .ant-tabs-tab { + background-color: @lightGray; + color: @darkBlue; + } + .ant-tabs-tab-active { + background-color: #0091FF; + color: #fff; + .ant-tabs-tab-btn { + color: #fff; + } + } + } + .ant-tabs-content-holder > .ant-tabs-content { + display: none + } +} \ No newline at end of file diff --git a/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx b/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx new file mode 100644 index 00000000..c7bc7448 --- /dev/null +++ b/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx @@ -0,0 +1,131 @@ +import { Button, Modal, Tabs } from 'antd'; +import _ from 'lodash'; +import React, { useEffect, useRef, useState } from 'react'; +import intl from 'react-intl-universal'; +import Icon from '@app/components/Icon'; +import './index.less'; +import { useStore } from '@app/stores'; +import { ITaskStatus } from '@app/interfaces/import'; + +const { TabPane } = Tabs; + +interface ILogDimension { + space: string; + id: number; + status: ITaskStatus; +} + +interface ILog { + name: string; + path: string; +} +interface IProps { + logDimension: ILogDimension; + visible: boolean; + onCancel: () => void; +} +const LogModal = (props: IProps) => { + const { visible, onCancel, logDimension: { space, id, status } } = props; + const { dataImport: { getLogs, downloadTaskLog, getImportLogDetail, getErrLogDetail } } = useStore(); + const logRef = useRef(null); + const timer = useRef(null); + const offset = useRef(0); + const _status = useRef(status); + const [logs, setLogs] = useState([]); + const [currentLog, setCurrentLog] = useState(null); + const handleTabChange = (key: string) => { + setCurrentLog(logs.filter(item => item.name === key)[0]); + }; + + const getAllLogs = async() => { + const { code, data } = await getLogs(id); + if(code === 0) { + setLogs(data); + setCurrentLog(data[0]); + } + }; + + const handleLogDownload = () => { + if(currentLog) { + downloadTaskLog(currentLog.path); + } + }; + + const readLog = async() => { + const getLogDetail = currentLog!.name === 'import.log' ? getImportLogDetail : getErrLogDetail; + const res = await getLogDetail({ + offset: offset.current, + taskId: id, + path: currentLog!.path + }); + handleLogData(res); + }; + + const handleLogData = (res) => { + const { data } = res; + if(!logRef.current) { + timer.current = setTimeout(readLog, 2000); + return; + } + if (data) { + logRef.current.innerHTML += data.join('
') + '
'; + logRef.current.scrollTop = logRef.current.scrollHeight; + offset.current += data.length; + timer.current = setTimeout(readLog, 2000); + } else if (_status.current === ITaskStatus.StatusProcessing) { + timer.current = setTimeout(readLog, 2000); + } else { + offset.current = 0; + } + }; + + useEffect(() => { + getAllLogs(); + return () => { + clearTimeout(timer.current); + }; + }, []); + + useEffect(() => { + _status.current = status; + }, [status]); + useEffect(() => { + clearTimeout(timer.current); + if(logRef.current) { + logRef.current.innerHTML = ''; + } + offset.current = 0; + if(currentLog) { + readLog(); + } + }, [currentLog]); + return ( + +
+ {`${space} ${intl.get('import.task')} - ${intl.get('common.log')}`} + {status === ITaskStatus.StatusProcessing &&
+ + } + width="80%" + visible={visible} + onCancel={onCancel} + wrapClassName="log-modal" + destroyOnClose={true} + footer={false} + > + + {logs.map(log => ( + + ))} + +
+ + ); +}; + +export default LogModal; diff --git a/app/pages/Import/TaskList/TaskItem/index.less b/app/pages/Import/TaskList/TaskItem/index.less index bf4a8ac7..be8d1eff 100644 --- a/app/pages/Import/TaskList/TaskItem/index.less +++ b/app/pages/Import/TaskList/TaskItem/index.less @@ -38,6 +38,9 @@ & > span { margin-right: 6px } + .red { + color: @red + } } .err-info { color: @red diff --git a/app/pages/Import/TaskList/TaskItem/index.tsx b/app/pages/Import/TaskList/TaskItem/index.tsx index da3107e8..d2f71e02 100644 --- a/app/pages/Import/TaskList/TaskItem/index.tsx +++ b/app/pages/Import/TaskList/TaskItem/index.tsx @@ -6,12 +6,14 @@ import './index.less'; import { ITaskItem, ITaskStatus } from '@app/interfaces/import'; import dayjs from 'dayjs'; import { floor } from 'lodash'; +import { getFileSize } from '@app/utils/file'; import Icon from '@app/components/Icon'; interface IProps { data: ITaskItem; handleStop: (id: number) => void; handleDelete: (id: number) => void; handleDownload: (id: number) => void; + onViewLog: (id: number, space: string, taskStatus: ITaskStatus) => void; } @@ -39,30 +41,35 @@ const TaskItem = (props: IProps) => { space, taskID, name, - statsQuery: { totalCount, totalLine, numFailed, numReadFailed }, + stats: { totalImportedBytes, totalBytes, numFailed, numReadFailed }, taskStatus, taskMessage, updatedTime, createdTime }, + onViewLog, handleDownload, handleStop, handleDelete } = props; const [status, setStatus] = useState<'success' | 'active' | 'normal' | 'exception' | undefined>(undefined); const [extraMsg, setExtraMsg] = useState(''); + const addMsg = () => { + const info: string[] = []; + if(numFailed > 0) { + info.push(intl.get('import.notImported', { total: getFileSize(numFailed) })); + } + if(numReadFailed > 0) { + info.push(intl.get('import.readFailed', { total: getFileSize(numReadFailed) })); + } + info.length > 0 && setExtraMsg(info.join(', ')); + }; useEffect(() => { if(taskStatus === ITaskStatus.StatusFinished) { setStatus('success'); + addMsg(); } else if(taskStatus === ITaskStatus.StatusProcessing) { setStatus('active'); - const info: string[] = []; - if(numFailed > 0) { - info.push(intl.get('import.notImported', { numFailed })); - } - if(numReadFailed > 0) { - info.push(intl.get('import.readFailed', { numReadFailed })); - } - info.length > 0 && setExtraMsg(info.join(', ')); + addMsg(); } else { setStatus('exception'); if(taskMessage) { @@ -87,7 +94,7 @@ const TaskItem = (props: IProps) => { {taskStatus === ITaskStatus.StatusFinished && {intl.get('import.importCompleted')} - {extraMsg && ` (${extraMsg})`} + {extraMsg && ` (${extraMsg})`} } {taskStatus === ITaskStatus.StatusAborted && {intl.get('import.importFailed')} @@ -99,20 +106,21 @@ const TaskItem = (props: IProps) => {
- {taskStatus !== ITaskStatus.StatusFinished && `${totalCount} ${intl.get('import.lines')} / `} - {totalLine}{' '}{intl.get('import.lines')} + {taskStatus !== ITaskStatus.StatusFinished && `${getFileSize(totalImportedBytes)} / `} + {getFileSize(totalBytes)}{' '} {dayjs.duration(dayjs.unix(updatedTime).diff(dayjs.unix(createdTime))).format('HH:mm:ss')}
`${percent}%`} status={status} - percent={taskStatus !== ITaskStatus.StatusFinished ? floor(totalCount / totalLine * 100, 2) : 100} + percent={taskStatus !== ITaskStatus.StatusFinished ? floor(totalImportedBytes / totalBytes * 100, 2) : 100} strokeColor={status && COLOR_MAP[status]} />
- - + {/* */} + {taskStatus === ITaskStatus.StatusProcessing && void; + onImport: () => void; +} + +const TemplateModal = (props: IProps) => { + const { visible, onClose, onImport } = props; + const [form] = Form.useForm(); + const [loading, setLoading] = useState(false); + const [config, setConfig] = useState(''); + const { dataImport: { getTaskDir, taskDir, importTask }, files: { uploadDir, asyncGetUploadDir } } = useStore(); + useEffect(() => { + if(!uploadDir) { + asyncGetUploadDir(); + } + getTaskDir(); + }, []); + const handleFileImport = async({ file }) => { + await setLoading(true); + const content = await readFileContent(file); + const parseContent = yaml.load(content); + if(typeof parseContent === 'object') { + const _taskDir = taskDir.endsWith('/') ? taskDir : taskDir + '/'; + const _uploadDir = uploadDir.endsWith('/') ? uploadDir : uploadDir + '/'; + parseContent.logPath = _taskDir + parseContent.logPath; + parseContent.files.forEach(file => { + file.path = _uploadDir + file.path; + file.failDataPath = _taskDir + `err\${file.failDataPath}`; + }); + setConfig(JSON.stringify(parseContent, null, 2)); + form.setFieldsValue({ + content: JSON.stringify(parseContent, null, 2), + }); + } else { + return message.warning(intl.get('import.parseFailed')); + } + await setLoading(false); + }; + + const handleImport = async(values) => { + const code = await importTask(JSON.parse(values.content), values.name); + if(code === 0) { + message.success(intl.get('import.startImporting')); + onImport(); + } + onClose(); + }; + return ( + + {!config ? +

{intl.get('import.fileUploadRequired')}

+ false} onChange={handleFileImport} showUploadList={false} accept=".yaml, .yml"> +
+ +
+

{intl.get('import.uploadTemplate')}

+

+ {intl.get('import.uploadTemplateTip')} +

+ + :
+ + + + + + + + +