From ace61648d05324fa4d54e5252fbf854640a5124a Mon Sep 17 00:00:00 2001 From: "ben.ding" <104374143+veeding@users.noreply.github.com> Date: Mon, 9 May 2022 03:23:30 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=B8=8A=E4=BC=A0=E3=80=81?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E3=80=81=E9=A2=84=E8=A7=88=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=20(#206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server-v2/api/studio/etc/studio-api.yaml | 3 + .../api/studio/internal/config/config.go | 4 + .../handler/file/filedestroyhandler.go | 33 +++ .../handler/file/filesindexhandler.go | 17 ++ .../handler/file/fileuploadhandler.go | 17 ++ .../api/studio/internal/handler/routes.go | 21 ++ .../internal/logic/file/filedestroylogic.go | 29 ++ .../internal/logic/file/filesindexlogic.go | 29 ++ .../internal/logic/file/fileuploadlogic.go | 27 ++ server-v2/api/studio/internal/service/file.go | 266 ++++++++++++++++++ server-v2/api/studio/internal/types/types.go | 16 ++ server-v2/api/studio/restapi/file.api | 35 +++ server-v2/api/studio/restapi/studio.api | 1 + 13 files changed, 498 insertions(+) create mode 100644 server-v2/api/studio/internal/handler/file/filedestroyhandler.go create mode 100644 server-v2/api/studio/internal/handler/file/filesindexhandler.go create mode 100644 server-v2/api/studio/internal/handler/file/fileuploadhandler.go create mode 100644 server-v2/api/studio/internal/logic/file/filedestroylogic.go create mode 100644 server-v2/api/studio/internal/logic/file/filesindexlogic.go create mode 100644 server-v2/api/studio/internal/logic/file/fileuploadlogic.go create mode 100644 server-v2/api/studio/internal/service/file.go create mode 100644 server-v2/api/studio/restapi/file.api diff --git a/server-v2/api/studio/etc/studio-api.yaml b/server-v2/api/studio/etc/studio-api.yaml index e6ea25df..002e4cd9 100644 --- a/server-v2/api/studio/etc/studio-api.yaml +++ b/server-v2/api/studio/etc/studio-api.yaml @@ -1,8 +1,11 @@ Name: studio-api Host: 0.0.0.0 Port: 7002 +MaxBytes: 1073741824 Debug: Enable: false Auth: AccessSecret: "login_secret" AccessExpire: 7200 +File: + UploadDir: "./upload/" \ No newline at end of file diff --git a/server-v2/api/studio/internal/config/config.go b/server-v2/api/studio/internal/config/config.go index 22736096..3efe2fcf 100644 --- a/server-v2/api/studio/internal/config/config.go +++ b/server-v2/api/studio/internal/config/config.go @@ -11,4 +11,8 @@ type Config struct { AccessSecret string AccessExpire int64 } + + File struct { + UploadDir string + } } diff --git a/server-v2/api/studio/internal/handler/file/filedestroyhandler.go b/server-v2/api/studio/internal/handler/file/filedestroyhandler.go new file mode 100644 index 00000000..fc84a36a --- /dev/null +++ b/server-v2/api/studio/internal/handler/file/filedestroyhandler.go @@ -0,0 +1,33 @@ +// Code generated by goctl. DO NOT EDIT. +package file + +import ( + "net/http" + + "github.com/vesoft-inc/go-pkg/validator" + "github.com/vesoft-inc/nebula-studio/server/api/studio/pkg/ecode" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/file" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/rest/httpx" +) + +func FileDestroyHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req types.FileDestroyRequest + if err := httpx.Parse(r, &req); err != nil { + err = ecode.WithCode(ecode.ErrParam, err) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + if err := validator.Struct(req); err != nil { + svcCtx.ResponseHandler.Handle(w, r, nil, err) + return + } + + l := file.NewFileDestroyLogic(r.Context(), svcCtx) + err := l.FileDestroy(req) + svcCtx.ResponseHandler.Handle(w, r, nil, err) + } +} diff --git a/server-v2/api/studio/internal/handler/file/filesindexhandler.go b/server-v2/api/studio/internal/handler/file/filesindexhandler.go new file mode 100644 index 00000000..c8ce0aaf --- /dev/null +++ b/server-v2/api/studio/internal/handler/file/filesindexhandler.go @@ -0,0 +1,17 @@ +// Code generated by goctl. DO NOT EDIT. +package file + +import ( + "net/http" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/file" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" +) + +func FilesIndexHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := file.NewFilesIndexLogic(r.Context(), svcCtx) + data, err := l.FilesIndex() + svcCtx.ResponseHandler.Handle(w, r, data, err) + } +} diff --git a/server-v2/api/studio/internal/handler/file/fileuploadhandler.go b/server-v2/api/studio/internal/handler/file/fileuploadhandler.go new file mode 100644 index 00000000..e379481b --- /dev/null +++ b/server-v2/api/studio/internal/handler/file/fileuploadhandler.go @@ -0,0 +1,17 @@ +// Code generated by goctl. DO NOT EDIT. +package file + +import ( + "net/http" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/logic/file" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" +) + +func FileUploadHandler(svcCtx *svc.ServiceContext) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + l := file.NewFileUploadLogic(r, svcCtx) + err := l.FileUpload() + svcCtx.ResponseHandler.Handle(w, r, nil, err) + } +} diff --git a/server-v2/api/studio/internal/handler/routes.go b/server-v2/api/studio/internal/handler/routes.go index 2b0cbe8d..f47653c5 100644 --- a/server-v2/api/studio/internal/handler/routes.go +++ b/server-v2/api/studio/internal/handler/routes.go @@ -4,6 +4,7 @@ package handler import ( "net/http" + file "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/file" gateway "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/gateway" health "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/handler/health" "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" @@ -43,4 +44,24 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { }, rest.WithPrefix("/api-nebula/db"), ) + + server.AddRoutes( + []rest.Route{ + { + Method: http.MethodPost, + Path: "/api/file", + Handler: file.FileUploadHandler(serverCtx), + }, + { + Method: http.MethodDelete, + Path: "/api/file/:name", + Handler: file.FileDestroyHandler(serverCtx), + }, + { + Method: http.MethodGet, + Path: "/api/file", + Handler: file.FilesIndexHandler(serverCtx), + }, + }, + ) } diff --git a/server-v2/api/studio/internal/logic/file/filedestroylogic.go b/server-v2/api/studio/internal/logic/file/filedestroylogic.go new file mode 100644 index 00000000..b70b97f3 --- /dev/null +++ b/server-v2/api/studio/internal/logic/file/filedestroylogic.go @@ -0,0 +1,29 @@ +package file + +import ( + "context" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type FileDestroyLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewFileDestroyLogic(ctx context.Context, svcCtx *svc.ServiceContext) *FileDestroyLogic { + return &FileDestroyLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *FileDestroyLogic) FileDestroy(req types.FileDestroyRequest) error { + return service.NewFileService(nil, l.ctx, l.svcCtx).FileDestroy(req.Name) +} diff --git a/server-v2/api/studio/internal/logic/file/filesindexlogic.go b/server-v2/api/studio/internal/logic/file/filesindexlogic.go new file mode 100644 index 00000000..21c5d3d5 --- /dev/null +++ b/server-v2/api/studio/internal/logic/file/filesindexlogic.go @@ -0,0 +1,29 @@ +package file + +import ( + "context" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + + "github.com/zeromicro/go-zero/core/logx" +) + +type FilesIndexLogic struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext +} + +func NewFilesIndexLogic(ctx context.Context, svcCtx *svc.ServiceContext) *FilesIndexLogic { + return &FilesIndexLogic{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } +} + +func (l *FilesIndexLogic) FilesIndex() (resp *types.FilesIndexData, err error) { + return service.NewFileService(nil, l.ctx, l.svcCtx).FilesIndex() +} diff --git a/server-v2/api/studio/internal/logic/file/fileuploadlogic.go b/server-v2/api/studio/internal/logic/file/fileuploadlogic.go new file mode 100644 index 00000000..81f5eaf6 --- /dev/null +++ b/server-v2/api/studio/internal/logic/file/fileuploadlogic.go @@ -0,0 +1,27 @@ +package file + +import ( + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/service" + "net/http" + + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/zeromicro/go-zero/core/logx" +) + +type FileUploadLogic struct { + logx.Logger + r *http.Request + svcCtx *svc.ServiceContext +} + +func NewFileUploadLogic(r *http.Request, svcCtx *svc.ServiceContext) *FileUploadLogic { + return &FileUploadLogic{ + Logger: logx.WithContext(r.Context()), + r: r, + svcCtx: svcCtx, + } +} + +func (l *FileUploadLogic) FileUpload() error { + return service.NewFileService(l.r, nil, l.svcCtx).FileUpload() +} diff --git a/server-v2/api/studio/internal/service/file.go b/server-v2/api/studio/internal/service/file.go new file mode 100644 index 00000000..9fd1a8a2 --- /dev/null +++ b/server-v2/api/studio/internal/service/file.go @@ -0,0 +1,266 @@ +package service + +import ( + "bufio" + "context" + "encoding/csv" + "errors" + "fmt" + "github.com/axgle/mahonia" + "github.com/saintfish/chardet" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/svc" + "github.com/vesoft-inc/nebula-studio/server/api/studio/internal/types" + "github.com/zeromicro/go-zero/core/logx" + "go.uber.org/zap" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strings" +) + +const ( + defaultMulipartMemory = 500 << 20 // 500 MB +) + +var ( + noCharsetErr = errors.New("this charset can not be changed") + _ FileService = (*fileService)(nil) +) + +type ( + FileService interface { + FileUpload() error + FileDestroy(string) error + FilesIndex() (*types.FilesIndexData, error) + } + + fileService struct { + logx.Logger + ctx context.Context + svcCtx *svc.ServiceContext + r *http.Request + } +) + +func NewFileService(r *http.Request, ctx context.Context, svcCtx *svc.ServiceContext) FileService { + if r != nil { + return &fileService{ + Logger: logx.WithContext(r.Context()), + r: r, + svcCtx: svcCtx, + } + } else { + return &fileService{ + Logger: logx.WithContext(ctx), + ctx: ctx, + svcCtx: svcCtx, + } + } +} + +func (f *fileService) FileDestroy(name string) error { + dir := f.svcCtx.Config.File.UploadDir + target := filepath.Join(dir, name) + if _, err := os.Stat(target); err != nil { + logx.Infof("del file error %v", err) + return err + } + + // if target is directory, it is not empty + if err := os.Remove(target); err != nil { + logx.Infof("del file error %v", err) + return err + } + + return nil +} + +func (f *fileService) FilesIndex() (data *types.FilesIndexData, err error) { + data = &types.FilesIndexData{ + List: []types.FileStat{}, + } + dir := f.svcCtx.Config.File.UploadDir + filesInfo, err := ioutil.ReadDir(dir) + if err != nil { + logx.Infof("open files error %v", err) + return nil, err + } + + for _, fileInfo := range filesInfo { + if fileInfo.IsDir() { + continue + } + path := filepath.Join(dir, fileInfo.Name()) + file, err := os.Open(path) + if err != nil { + logx.Infof("open files error %v", err) + continue + } + reader := csv.NewReader(file) + count := 0 + content := make([][]string, 0) + for count < 3 { + line, err := reader.Read() + count++ + if err != nil { + break + } + content = append(content, line) + } + data.List = append(data.List, types.FileStat{ + Content: content, + DataType: "all", + WithHeader: false, + Name: fileInfo.Name(), + Size: fileInfo.Size(), + }) + } + return data, nil +} + +func (f *fileService) FileUpload() error { + dir := f.svcCtx.Config.File.UploadDir + _, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + os.MkdirAll(dir, os.ModePerm) + } else { + return err + } + } + + logx.Infof("dir:", dir) + files, _, err := f.UploadFormFiles(dir) + if err != nil { + logx.Infof("upload file error:%v", err) + return err + } + for _, file := range files { + charSet, err := checkCharset(file) + if err != nil { + logx.Infof("upload file error, check charset fail:%v", err) + return err + } + if charSet == "UTF-8" { + continue + } + path := filepath.Join(dir, file.Filename) + if err = changeFileCharset2UTF8(path, charSet); err != nil { + logx.Infof("upload file error:%v", err) + return err + } + } + logx.Infof("upload %d files", len(files)) + return nil +} + +func (f *fileService) UploadFormFiles(destDirectory string) (uploaded []*multipart.FileHeader, n int64, err error) { + err = f.r.ParseMultipartForm(defaultMulipartMemory) + if err != nil { + return nil, 0, err + } + + if f.r.MultipartForm != nil { + if fhs := f.r.MultipartForm.File; fhs != nil { + for _, files := range fhs { + for _, file := range files { + file.Filename = strings.ReplaceAll(file.Filename, "../", "") + file.Filename = strings.ReplaceAll(file.Filename, "..\\", "") + + n0, err0 := f.SaveFormFile(file, filepath.Join(destDirectory, file.Filename)) + if err0 != nil { + return nil, 0, err0 + } + n += n0 + + uploaded = append(uploaded, file) + } + } + return uploaded, n, nil + } + } + return nil, 0, http.ErrMissingFile +} + +func (f *fileService) SaveFormFile(fh *multipart.FileHeader, dest string) (int64, error) { + src, err := fh.Open() + if err != nil { + return 0, err + } + defer src.Close() + + out, err := os.Create(dest) + if err != nil { + return 0, err + } + defer out.Close() + + return io.Copy(out, src) +} + +func checkCharset(file *multipart.FileHeader) (string, error) { + f, err := file.Open() + if err != nil { + return "", err + } + defer f.Close() + bytes := make([]byte, 1024) + if _, err = f.Read(bytes); err != nil { + return "", err + } + detector := chardet.NewTextDetector() + best, err := detector.DetectBest(bytes) + if err != nil { + return "", err + } + return best.Charset, nil +} + +func changeFileCharset2UTF8(filePath string, charSet string) error { + fileUTF8Path := filePath + "-copy" + err := func() error { + file, err := os.OpenFile(filePath, os.O_RDONLY, 0666) + if err != nil { + zap.L().Warn("open file fail", zap.Error(err)) + return err + } + defer file.Close() + reader := bufio.NewReader(file) + decoder := mahonia.NewDecoder(charSet) + // this charset can not be changed + if decoder == nil { + return noCharsetErr + } + decodeReader := decoder.NewReader(reader) + fileUTF8, err := os.OpenFile(fileUTF8Path, os.O_RDONLY|os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return err + } + defer fileUTF8.Close() + writer := bufio.NewWriter(fileUTF8) + if _, err = writer.ReadFrom(decodeReader); err != nil { + return err + } + return nil + }() + if err != nil { + _, statErr := os.Stat(fileUTF8Path) + if statErr == nil || os.IsExist(statErr) { + removeErr := os.Remove(fileUTF8Path) + if removeErr != nil { + zap.L().Warn(fmt.Sprintf("remove file %s fail", fileUTF8Path), zap.Error(removeErr)) + } + } + if err == noCharsetErr { + return nil + } + return err + } + if err = os.Rename(fileUTF8Path, filePath); err != nil { + return err + } + return nil +} diff --git a/server-v2/api/studio/internal/types/types.go b/server-v2/api/studio/internal/types/types.go index aa2e03fe..373f0ca5 100644 --- a/server-v2/api/studio/internal/types/types.go +++ b/server-v2/api/studio/internal/types/types.go @@ -23,3 +23,19 @@ type ConnectDBResult struct { type AnyResponse struct { Data interface{} `json:"data"` } + +type FileDestroyRequest struct { + Name string `path:"name" validate:"required"` +} + +type FileStat struct { + Content [][]string `json:"content"` + WithHeader bool `json:"withHeader"` + DataType string `json:"dataType"` + Name string `json:"name"` + Size int64 `json:"size"` +} + +type FilesIndexData struct { + List []FileStat `json:"list"` +} diff --git a/server-v2/api/studio/restapi/file.api b/server-v2/api/studio/restapi/file.api new file mode 100644 index 00000000..f4ae3b71 --- /dev/null +++ b/server-v2/api/studio/restapi/file.api @@ -0,0 +1,35 @@ +syntax = "v1" + +type ( + FileDestroyRequest { + Name string `path:"name" validate:"required"` + } + + FileStat { + Content [][]string `json:"content"` + WithHeader bool `json:"withHeader"` + DataType string `json:"dataType"` + Name string `json:"name"` + Size int64 `json:"size"` + } + + FilesIndexData { + List []FileStat `json:"list"` + } +) + +@server( + group: file +) + +service studio-api { + @doc "Upload File" + @handler FileUpload + post /api/file + @doc "delete file" + @handler FileDestroy + delete /api/file/:name returns(FileDestroyRequest) + @doc "preview file" + @handler FilesIndex + get /api/file returns(FilesIndexData) +} \ No newline at end of file diff --git a/server-v2/api/studio/restapi/studio.api b/server-v2/api/studio/restapi/studio.api index c7bbf924..9c362460 100644 --- a/server-v2/api/studio/restapi/studio.api +++ b/server-v2/api/studio/restapi/studio.api @@ -8,4 +8,5 @@ info( import ( "health.api" "gateway.api" + "file.api" ) \ No newline at end of file