feat: store task effects in db (#643)

- feat: allowClear for task select - feat: store task effects in db
vesoft-inc · Sep 18, 2023 · 4a23e9b · 4a23e9b
1 parent 507afa4
commit 4a23e9b
Show file tree

Hide file tree

Showing 14 changed files with 270 additions and 195 deletions.
diff --git a/app/pages/Import/TaskList/TaskItem/LogModal/index.module.less b/app/pages/Import/TaskList/TaskItem/LogModal/index.module.less
@@ -50,11 +50,15 @@
     text-align: left;
     background: #333;
     color: #fff;
-    word-break: break-all;
+    white-space: nowrap;
   }
   .full {
     width: 100%;
   }
+  code {
+    font-size: 12px;
+    font-family: inherit;
+  }
 }
 .logTab {
   height: 100%;

diff --git a/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx b/app/pages/Import/TaskList/TaskItem/LogModal/index.tsx
@@ -32,13 +32,10 @@ const LogModal = (props: IProps) => {
   const { disableLogDownload } = moduleConfiguration.dataImport;
   const { intl } = useI18n();
   const logRef = useRef<HTMLDivElement>(null);
-  const timer = useRef<any>(null);
-  const offset = useRef(0);
-  const isMounted = useRef(true);
-  const _status = useRef(status);
   const [logs, setLogs] = useState<string[]>([]);
   const [loading, setLoading] = useState(false);
   const [currentLog, setCurrentLog] = useState<string | null>(null);
+
   const handleTabChange = (key: string) => {
     setCurrentLog(logs.filter((item) => item === key)[0]);
   };
@@ -52,83 +49,59 @@ const LogModal = (props: IProps) => {
     }
   };
 
-  const handleLogDownload = () => {
-    if (currentLog) {
-      downloadTaskLog({
-        id,
-        name: currentLog,
-      });
-    }
-  };
+  const handleLogDownload = () => currentLog && downloadTaskLog({ id, name: currentLog });
 
   const readLog = async () => {
-    const data = await getLogDetail({
-      start: offset.current,
-      id,
-      end: [ITaskStatus.Finished, ITaskStatus.Aborted, ITaskStatus.Stoped].includes(_status.current)
-        ? 0
-        : offset.current + 4096, // 4kb content per request, if task is finished, read all logs
-      file: currentLog,
-    });
-    isMounted.current && handleLogData(data);
+    const data = await getLogDetail({ id });
+    handleLogData(data);
   };
 
   const handleLogData = (data) => {
-    if (!logRef.current) {
-      timer.current = setTimeout(readLog, 2000);
+    const logs = data?.logs || '';
+    if (!logs.length) {
       return;
     }
-    const { logs, endPosition } = data;
-    if (logs.length > 0) {
-      logRef.current.innerHTML += logs.split('\n').join('<br/>');
-      logRef.current.scrollTop = logRef.current.scrollHeight;
-      offset.current = endPosition;
-      if (isMounted.current) {
-        timer.current = setTimeout(readLog, 2000);
-      }
-    } else if ([ITaskStatus.Processing, ITaskStatus.Pending].includes(_status.current)) {
-      if (isMounted.current) {
-        timer.current = setTimeout(readLog, 2000);
-      }
-    } else {
-      offset.current = 0;
-    }
+    /**
+     * {"level":"info",...}
+     * {"level":"info",...}
+     *
+     * ...
+     * (200 lines more, original log file path: /.../tasks/nck3vu9b7id2r67lvi1b0/import.log, hostname: xxx)
+     * ...
+     *
+     * {"level":"info",...}
+     * {"level":"info",...}
+     */
+    logRef.current.innerHTML = logs
+      .split('\n')
+      .map((log) => `<code style="color:${/^(\.\.\.)|^\(\d+/.test(log) ? '#fff' : '#e8c18b'}">${log}</code>`)
+      .join('<br/>');
+    logRef.current.scrollTop = logRef.current.scrollHeight;
   };
 
   const initLog = async () => {
     setLoading(true);
     await readLog();
     setLoading(false);
   };
+
   useEffect(() => {
     if (!disableLogDownload) {
       getAllLogs();
     } else {
       initLog();
     }
-    return () => {
-      isMounted.current = false;
-      clearTimeout(timer.current);
-    };
   }, []);
 
   useEffect(() => {
-    _status.current = status;
-  }, [status]);
-  useEffect(() => {
-    clearTimeout(timer.current);
     if (logRef.current) {
       logRef.current.innerHTML = '';
     }
-    offset.current = 0;
-    if (currentLog && isMounted.current) {
-      readLog();
-    }
+    currentLog && initLog();
   }, [currentLog]);
-  const items = logs.map((log) => ({
-    key: log,
-    label: log,
-  }));
+
+  const items = logs.map((log) => ({ key: log, label: log }));
+
   return (
     <Modal
       title={

diff --git a/app/pages/Import/TaskList/TaskItem/index.tsx b/app/pages/Import/TaskList/TaskItem/index.tsx
@@ -235,7 +235,7 @@ const TaskItem = (props: IProps) => {
                 <Icon type="icon-studio-btn-edit" />
               </Tooltip>
             </Button>
-            {!isDraft && (
+            {!isDraft && !loadingStatus.includes(status) && (
               <Button className="primaryBtn" onClick={() => onViewLog(id, space, status)}>
                 <Tooltip title={intl.get('import.viewLogs')}>
                   <Icon type="icon-studio-btn-ddl" />

diff --git a/app/pages/Schema/SchemaConfig/List/SpaceStats/index.tsx b/app/pages/Schema/SchemaConfig/List/SpaceStats/index.tsx
@@ -13,35 +13,40 @@ import styles from './index.module.less';
 const SpaceStats = () => {
   const timer = useRef<NodeJS.Timeout | null>(null);
   const { intl } = useI18n();
-  const { schema: { getJobStatus, submitStats, getStats, currentSpace } } = useStore();
+  const {
+    schema: { getJobStatus, submitStats, getStats, currentSpace },
+  } = useStore();
   const [data, setData] = useState<{
     list: {
-      Type: string,
-      Name: string,
-      Count: number,
-    }[],
+      Type: string;
+      Name: string;
+      Count: number;
+    }[];
     total?: {
-      vertices: number,
-      edges: number,
-    }
+      vertices: number;
+      edges: number;
+    };
   }>({ list: [], total: undefined });
   const [updateTime, setUpdateTime] = useState('');
   const [jobId, setJobId] = useState<any>(null);
   const [loading, setLoading] = useState(false);
-  const columns = useMemo(() => [
-    {
-      title: intl.get('schema.statsType'),
-      dataIndex: 'Type',
-    },
-    {
-      title: intl.get('schema.statsName'),
-      dataIndex: 'Name',
-    },
-    {
-      title: intl.get('schema.statsCount'),
-      dataIndex: 'Count',
-    },
-  ], [Cookie.get('lang')]);
+  const columns = useMemo(
+    () => [
+      {
+        title: intl.get('schema.statsType'),
+        dataIndex: 'Type',
+      },
+      {
+        title: intl.get('schema.statsName'),
+        dataIndex: 'Name',
+      },
+      {
+        title: intl.get('schema.statsCount'),
+        dataIndex: 'Count',
+      },
+    ],
+    [Cookie.get('lang')],
+  );
   useEffect(() => {
     trackPageView('/space/stats');
     initData();
@@ -56,30 +61,33 @@ const SpaceStats = () => {
     setUpdateTime('');
     setData({
       list: [],
-      total: undefined
+      total: undefined,
     });
   };
 
   const getData = async () => {
     const { code, data } = await getStats();
     if (code === 0) {
-      const _data = data.tables.reduce((prev, cur) => {
-        if (cur.Type === 'Space') {
-          prev.total ||= { vertex: 0, edge: 0 };
-          prev.total[cur.Name] = cur.Count;
-        } else {
-          prev.list.push(cur);
-        }
-        return prev;
-      }, { list: [], total: undefined });
+      const _data = data.tables.reduce(
+        (prev, cur) => {
+          if (cur.Type === 'Space') {
+            prev.total ||= { vertex: 0, edge: 0 };
+            prev.total[cur.Name] = cur.Count;
+          } else {
+            prev.list.push(cur);
+          }
+          return prev;
+        },
+        { list: [], total: undefined },
+      );
       setData(_data);
     }
   };
 
   const getJobs = async () => {
     const { code, data } = await getJobStatus();
     if (code === 0) {
-      const stat = data.tables.filter(item => item.Command === 'STATS')[0];
+      const stat = data.tables.filter((item) => item.Command === 'STATS')[0];
       if (stat?.Status === IJobStatus.Finished) {
         getData();
         setUpdateTime(stat['Stop Time']);
@@ -119,36 +127,35 @@ const SpaceStats = () => {
     }
   };
   const showTime = updateTime && jobId == null && !loading;
+  console.log('=====updateTime', updateTime);
   return (
     <div className={styles.nebulaStats}>
       <div className={styles.row}>
         <div className={styles.operations}>
-          <Button
-            type="primary"
-            onClick={handleSubmitStats}
-            loading={loading || jobId !== null}
-          >
+          <Button type="primary" onClick={handleSubmitStats} loading={loading || jobId !== null}>
             {intl.get(updateTime ? 'schema.refresh' : 'schema.startStat')}
           </Button>
-          {showTime ? <>
-            <span className={styles.label}>{intl.get('schema.lastRefreshTime')}</span>
-            <span>
-              {dayjs(updateTime).format('YYYY-MM-DD HH:mm:ss')}
-            </span>
-          </> : <span className={styles.tip}>
-            {intl.get('schema.statTip')}
-          </span>}
+          {showTime ? (
+            <>
+              <span className={styles.label}>{intl.get('schema.lastRefreshTime')}</span>
+              <span>{dayjs(updateTime).format('YYYY-MM-DD HH:mm:ss')}</span>
+            </>
+          ) : (
+            <span className={styles.tip}>{intl.get('schema.statTip')}</span>
+          )}
         </div>
-        {data?.total && <div className={styles.totalCount}>
-          <div className={styles.totalItem}>
-            <span className={styles.label}>{intl.get('schema.totalVertices')}</span>
-            <span>{data.total.vertices}</span>
-          </div>
-          <div className={styles.totalItem}>
-            <span className={styles.label}>{intl.get('schema.totalEdges')}</span>
-            <span>{data.total.edges}</span>
+        {data?.total && (
+          <div className={styles.totalCount}>
+            <div className={styles.totalItem}>
+              <span className={styles.label}>{intl.get('schema.totalVertices')}</span>
+              <span>{data.total.vertices}</span>
+            </div>
+            <div className={styles.totalItem}>
+              <span className={styles.label}>{intl.get('schema.totalEdges')}</span>
+              <span>{data.total.edges}</span>
+            </div>
           </div>
-        </div>}
+        )}
       </div>
       <Table
         dataSource={data?.list || []}

diff --git a/app/stores/import.ts b/app/stores/import.ts
@@ -341,7 +341,7 @@ export class ImportStore {
     trackEvent('import', 'download_task_log');
   };
 
-  getLogDetail = async (params: { start: number; end: number; id: string; file: string }) => {
+  getLogDetail = async (params: { id: string }) => {
     const { code, data } = await service.getLogDetail(params);
     if (code === 0) {
       return data;

diff --git a/server/api/studio/internal/config/config.go b/server/api/studio/internal/config/config.go
@@ -9,6 +9,12 @@ import (
 	"github.com/zeromicro/go-zero/rest"
 )
 
+var configIns *Config
+
+func GetConfig() *Config {
+	return configIns
+}
+
 type Config struct {
 	rest.RestConf
 	Debug struct {
@@ -88,5 +94,7 @@ func (c *Config) Complete() {
 func (c *Config) InitConfig() error {
 	c.Complete()
 
+	configIns = c
+
 	return c.Validate()
 }
diff --git a/server/api/studio/internal/model/db.go b/server/api/studio/internal/model/db.go
@@ -107,6 +107,7 @@ func InitDB(config *config.Config, db *gorm.DB) {
 		err := db.AutoMigrate(
 			&Datasource{},
 			&TaskInfo{},
+			&TaskEffect{},
 			&Sketch{},
 			&SchemaSnapshot{},
 			&Favorite{},

diff --git a/server/api/studio/internal/model/taskInfo.go b/server/api/studio/internal/model/taskInfo.go
@@ -16,6 +16,7 @@ type Stats struct {
 	FailedProcessed int64         `gorm:"column:failed_processed;"`
 	TotalProcessed  int64         `gorm:"column:total_processed;"`
 }
+
 type TaskInfo struct {
 	ID            int    `gorm:"column:id;primaryKey;autoIncrement;"`
 	BID           string `gorm:"column:b_id;not null;type:char(32);uniqueIndex;comment:task id"`
@@ -32,3 +33,13 @@ type TaskInfo struct {
 	CreateTime time.Time `gorm:"column:create_time;type:datetime;autoCreateTime"`
 	UpdateTime time.Time `gorm:"column:update_time;type:datetime;autoUpdateTime"`
 }
+
+// storage for task yaml config and partial task log
+type TaskEffect struct {
+	ID     int    `gorm:"column:id;primaryKey;autoIncrement;"`
+	BID    string `gorm:"column:task_id;not null;type:char(32);uniqueIndex;comment:task id"`
+	Log    string `gorm:"column:log;type:mediumtext;comment:partial task log"`
+	Config string `gorm:"column:config;type:mediumtext;comment:task config.yaml"`
+
+	CreateTime time.Time `gorm:"column:create_time;type:datetime;autoCreateTime"`
+}