diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json index 09058ae8..13f6a47b 100644 --- a/app/config/locale/en-US.json +++ b/app/config/locale/en-US.json @@ -220,7 +220,7 @@ "importYaml": "Import the YAML file", "templateMatchError": "The user in the configuration does not match the current login account", "uploadSuccessfully": "Upload files successfully.", - "fileSizeLimit": "The file is too large and exceeds the upload limit({size}), please upload the file to the data/upload directory under the installation directory via scp", + "fileSizeLimit": "{name} is too large and exceeds the upload limit({size}), please upload the file to the data/upload directory under the installation directory via scp", "noHttp": "The address in the configuration file does not support http protocol, please remove http(s)://", "addressMatch": "The address in the configuration file must contain the Graph address of current login connection. Separate multiple addresses with ", "dataSourceFile": "Data source file", diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json index 928154d7..6360f2f8 100644 --- a/app/config/locale/zh-CN.json +++ b/app/config/locale/zh-CN.json @@ -220,7 +220,7 @@ "importYaml": "导入 YAML 文件", "templateMatchError": "配置中的用户姓名与当前登录账号不一致", "uploadSuccessfully": "上传文件成功", - "fileSizeLimit": "文件过大,超过上传限制({size}),请将文件通过 scp 的方式上传到安装目录下的 data/upload 目录", + "fileSizeLimit": "{name}文件过大,超过上传限制({size}),请将文件通过 scp 的方式上传到安装目录下的 data/upload 目录", "noHttp": "配置文件中的 address 不支持携带 http 协议,请去除 http(s)://", "addressMatch": "配置文件中的 address 字段必须包含当前登录的 Graph 地址。多个地址用“,”隔开。", "dataSourceFile": "文件源", diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx index f3f89b26..abbc5361 100644 --- a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx +++ b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx @@ -21,9 +21,9 @@ const UploadBtn = (props: IUploadBtnProps) => { const { fileList, uploadFile } = files; const [visible, setVisible] = useState(false); const transformFile = async (_file: StudioFile, fileList: StudioFile[]) => { - const size = fileList.reduce((acc, cur) => acc + cur.size, 0); - if(size > SizeLimit) { - message.error(intl.get('import.fileSizeLimit', { size: getFileSize(SizeLimit) })); + const bigFiles = fileList.filter(file => file.size > SizeLimit); + if(bigFiles.length > 0) { + message.error(intl.get('import.fileSizeLimit', { name: bigFiles.map(i => i.name).join(', '), size: getFileSize(SizeLimit) })); return false; } fileList.forEach(file => { diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx index 92fae6a1..a8d95a02 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx +++ b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx @@ -93,8 +93,9 @@ const FileSelect = observer((props: IFileSelect) => { } }, []); const handleRefresh = useCallback(async () => { + if (!activeId) return; setState({ loading: true }); - if (!activeId || activeId === IDatasourceType.Local) { + if (activeId === IDatasourceType.Local) { getLocalFiles(); return; } diff --git a/app/pages/Import/TaskCreate/SchemaConfig/index.module.less b/app/pages/Import/TaskCreate/SchemaConfig/index.module.less index 23bc03a8..91d30642 100644 --- a/app/pages/Import/TaskCreate/SchemaConfig/index.module.less +++ b/app/pages/Import/TaskCreate/SchemaConfig/index.module.less @@ -152,13 +152,15 @@ } .concatItems { background: #F3F6F9; - padding: 8px; + padding: 8px 8px 0; flex: 1; .tagItem { background: #8697B0; border-radius: 3px; padding: 6px; color: #fff; + margin-bottom: 10px; + display: inline-block; &:not(:last-child) { margin-right: 10px; } diff --git a/app/pages/Import/TaskList/TemplateModal/index.tsx b/app/pages/Import/TaskList/TemplateModal/index.tsx index 09025242..e760844d 100644 --- a/app/pages/Import/TaskList/TemplateModal/index.tsx +++ b/app/pages/Import/TaskList/TemplateModal/index.tsx @@ -176,7 +176,7 @@ const TemplateModal = (props: IProps) => { const handleImport = async (values) => { const code = await importTask({ - config: JSON.parse(values.content), + config: values.content, name: values.name }); if(code === 0) { diff --git a/app/utils/import.ts b/app/utils/import.ts index 8f3e8d77..045b6ca3 100644 --- a/app/utils/import.ts +++ b/app/utils/import.ts @@ -53,7 +53,7 @@ export function configToJson(payload: IConfig) { }, sources }; - return configJson; + return JSON.stringify(configJson); } const getIdConfig = (payload: { @@ -69,7 +69,9 @@ const getIdConfig = (payload: { function: vidFunction, } as any; if(indexes.length > 1 || !!prefix || !!suffix) { - id.concatItems = [prefix, ...indexes, suffix]; + id.concatItems = [...indexes]; + prefix && id.concatItems.unshift(prefix); + suffix && id.concatItems.push(suffix); } else { id.index = indexes[0]; } diff --git a/server/api/studio/internal/service/datasource.go b/server/api/studio/internal/service/datasource.go index 9ab3c9b1..e17b6c29 100644 --- a/server/api/studio/internal/service/datasource.go +++ b/server/api/studio/internal/service/datasource.go @@ -208,6 +208,7 @@ func (d *datasourceService) ListContents(request types.DatasourceListContentsReq return nil, err } fileList, err := store.ListFiles(request.Path) + defer store.Close() list := make([]types.FileConfig, 0) for _, item := range fileList { list = append(list, types.FileConfig{ @@ -235,6 +236,7 @@ func (d *datasourceService) PreviewFile(request types.DatasourcePreviewFileReque } // read three lines contents, err := store.ReadFile(request.Path, 0, 4) + defer store.Close() if err != nil { return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "readFiles failed") } diff --git a/server/api/studio/internal/service/import.go b/server/api/studio/internal/service/import.go index 903efc1b..4a526054 100644 --- a/server/api/studio/internal/service/import.go +++ b/server/api/studio/internal/service/import.go @@ -68,7 +68,11 @@ func NewImportService(ctx context.Context, svcCtx *svc.ServiceContext) ImportSer } func (i *importService) updateDatasourceConfig(conf *types.CreateImportTaskRequest) (*types.ImportTaskConfig, error) { - config := conf.Config + // config := conf.Config + var config types.ImportTaskConfig + if err := json.Unmarshal([]byte(conf.Config), &config); err != nil { + return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err) + } for _, source := range config.Sources { if source.DatasourceId != nil { var dbs db.Datasource diff --git a/server/api/studio/internal/types/types.go b/server/api/studio/internal/types/types.go index 07166e88..3279e3fa 100644 --- a/server/api/studio/internal/types/types.go +++ b/server/api/studio/internal/types/types.go @@ -180,8 +180,8 @@ type LogField struct { } type CreateImportTaskRequest struct { - Name string `json:"name" validate:"required"` - Config ImportTaskConfig `json:"config" validate:"required"` + Name string `json:"name" validate:"required"` + Config string `json:"config" validate:"required"` } type CreateImportTaskData struct { diff --git a/server/api/studio/pkg/filestore/filestore.go b/server/api/studio/pkg/filestore/filestore.go index 3f70fadb..4f0d9ba6 100644 --- a/server/api/studio/pkg/filestore/filestore.go +++ b/server/api/studio/pkg/filestore/filestore.go @@ -9,6 +9,7 @@ type ( FileStore interface { ReadFile(path string, startLine ...int) ([]string, error) ListFiles(dir string) ([]FileConfig, error) + Close() error } FileConfig struct { diff --git a/server/api/studio/pkg/filestore/s3store.go b/server/api/studio/pkg/filestore/s3store.go index 5e39b3ed..5bddbb7c 100644 --- a/server/api/studio/pkg/filestore/s3store.go +++ b/server/api/studio/pkg/filestore/s3store.go @@ -156,3 +156,7 @@ func (s *S3Store) ListBuckets() ([]string, error) { return buckets, nil } + +func (s *S3Store) Close() error { + return nil +} diff --git a/server/api/studio/pkg/filestore/sftpstore.go b/server/api/studio/pkg/filestore/sftpstore.go index 1074170e..ca911e69 100644 --- a/server/api/studio/pkg/filestore/sftpstore.go +++ b/server/api/studio/pkg/filestore/sftpstore.go @@ -125,3 +125,7 @@ func (s *SftpStore) ListFiles(dir string) ([]FileConfig, error) { } return files, nil } + +func (s *SftpStore) Close() error { + return s.SftpClient.Close() +} diff --git a/server/api/studio/pkg/utils/datasourceParse.go b/server/api/studio/pkg/utils/datasourceParse.go index b7a9d5d9..3c69b94a 100644 --- a/server/api/studio/pkg/utils/datasourceParse.go +++ b/server/api/studio/pkg/utils/datasourceParse.go @@ -22,7 +22,6 @@ func ParseEndpoint(platform, rawEndpoint string) (string, string, error) { } host := u.Hostname() parts := strings.SplitN(host, ".", 2) - fmt.Println("host", host, parts) var ( bucket string endpoint string diff --git a/server/api/studio/restapi/import.api b/server/api/studio/restapi/import.api index 78fe1f6f..22bb1314 100644 --- a/server/api/studio/restapi/import.api +++ b/server/api/studio/restapi/import.api @@ -132,8 +132,8 @@ type ( } CreateImportTaskRequest { - Name string `json:"name" validate:"required"` - Config ImportTaskConfig `json:"config" validate:"required"` + Name string `json:"name" validate:"required"` + Config string `json:"config" validate:"required"` } CreateImportTaskData {