-
Notifications
You must be signed in to change notification settings - Fork 21
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Supports nebula-importer apis and add a custom logger * feat: Support importer timecost stat and remove chores * Add importer upload files support * Modify the logging format * Fix 404 routers * Remove redeclared struct * Migrate import and taskmgr into importer package * Replace nebula client logger with httpgateway logger adapter * Make filepath only be absolute * Update README.md * Optimize file create prems * Update README * Supports set import config body in Request Body * Handle config body unmarshal error * Update README * Update README * echo the task id after import && support query task detail status * Update README * remove chore dev logging * Add task not start err handle in * Support save task status to sqlite3 * remove * fix query action bug * use channel to wait server quit singal to make shutdown slowly * add code comments and optimize import actions Co-authored-by: Yee <[email protected]>
- Loading branch information
Showing
14 changed files
with
867 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package common | ||
|
|
||
| import ( | ||
| "errors" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "strconv" | ||
| "syscall" | ||
| ) | ||
|
|
||
| func CreateFileWithPerm(filePath string, permCode string) (*os.File, error) { | ||
|
|
||
| if abs := filepath.IsAbs(filePath); !abs { | ||
| return nil, errors.New("file path should be absolute path") | ||
| } | ||
|
|
||
| perm, err := strconv.ParseInt(permCode, 8, 64) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| mask := syscall.Umask(0) | ||
| defer syscall.Umask(mask) | ||
| filedir := path.Dir(filePath) | ||
| os.MkdirAll(filedir, os.FileMode(perm)) | ||
| fd, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(perm)) | ||
| if os.IsExist(err) { | ||
| os.Chmod(filePath, os.FileMode(perm)) | ||
| } | ||
| return fd, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| package controllers | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
|
|
||
| "github.com/astaxie/beego" | ||
| "github.com/vesoft-inc/nebula-http-gateway/service/importer" | ||
| "github.com/vesoft-inc/nebula-importer/pkg/config" | ||
|
|
||
| importerErrors "github.com/vesoft-inc/nebula-importer/pkg/errors" | ||
| ) | ||
|
|
||
| type TaskController struct { | ||
| beego.Controller | ||
| } | ||
|
|
||
| type ImportRequest struct { | ||
| ConfigPath string `json:"configPath"` | ||
| ConfigBody config.YAMLConfig `json:"configBody"` | ||
| } | ||
|
|
||
| type ImportActionRequest struct { | ||
| TaskID string `json:"taskID"` | ||
| TaskAction string `json:"taskAction"` | ||
| } | ||
|
|
||
| func (this *TaskController) Import() { | ||
| var ( | ||
| res Response | ||
| params ImportRequest | ||
| taskID string = importer.NewTaskID() | ||
| err error | ||
| ) | ||
|
|
||
| task := importer.NewTask(taskID) | ||
| importer.GetTaskMgr().PutTask(taskID, &task) | ||
|
|
||
| err = json.Unmarshal(this.Ctx.Input.RequestBody, ¶ms) | ||
|
|
||
| if err != nil { | ||
| err = importerErrors.Wrap(importerErrors.InvalidConfigPathOrFormat, err) | ||
| } else { | ||
| err = importer.Import(taskID, params.ConfigPath, ¶ms.ConfigBody) | ||
| } | ||
|
|
||
| if err != nil { | ||
| // task err: import task not start err handle | ||
| task.TaskStatus = importer.StatusAborted.String() | ||
| beego.Error(fmt.Sprintf("Failed to start a import task: `%s`, task result: `%v`", taskID, err)) | ||
|
|
||
| res.Code = -1 | ||
| res.Message = err.Error() | ||
| } else { | ||
| res.Code = 0 | ||
| res.Data = []string{taskID} | ||
| res.Message = fmt.Sprintf("Import task %s submit successfully", taskID) | ||
| } | ||
| this.Data["json"] = &res | ||
| this.ServeJSON() | ||
| } | ||
|
|
||
| func (this *TaskController) ImportAction() { | ||
| var res Response | ||
| var params ImportActionRequest | ||
|
|
||
| json.Unmarshal(this.Ctx.Input.RequestBody, ¶ms) | ||
| result, err := importer.ImportAction(params.TaskID, importer.NewTaskAction(params.TaskAction)) | ||
| if err == nil { | ||
| res.Code = 0 | ||
| res.Data = result | ||
| res.Message = "Processing a task action successfully" | ||
| } else { | ||
| res.Code = -1 | ||
| res.Message = err.Error() | ||
| } | ||
| this.Data["json"] = &res | ||
| this.ServeJSON() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| package controllers | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "io/ioutil" | ||
| "log" | ||
| "net/http" | ||
| "testing" | ||
| ) | ||
|
|
||
| func Test_Task_Import(t *testing.T) { | ||
| /* | ||
| */ | ||
| cases := []struct { | ||
| path string | ||
| requestMethod string | ||
| requestBody []byte | ||
| }{ | ||
| { | ||
| "http://127.0.0.1:8080/api/task/import", | ||
| "POST", | ||
| []byte(`{"configPath" : "examples/v2/example.yaml"}`), | ||
| }, | ||
| } | ||
| for _, tc := range cases { | ||
| var Response Response | ||
| req, err := http.NewRequest(tc.requestMethod, tc.path, bytes.NewBuffer(tc.requestBody)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
|
|
||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
|
|
||
| client := &http.Client{} | ||
| resp, err := client.Do(req) | ||
|
|
||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
|
|
||
| defer resp.Body.Close() | ||
|
|
||
| body, _ := ioutil.ReadAll(resp.Body) | ||
|
|
||
| json.Unmarshal([]byte(body), &Response) | ||
| if Response.Code != -1 && Response.Code != 0 { | ||
| t.Fail() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func Test_Task_Action(t *testing.T) { | ||
| /* | ||
| */ | ||
| cases := []struct { | ||
| path string | ||
| requestMethod string | ||
| requestBody []byte | ||
| }{ | ||
| { | ||
| "http://127.0.0.1:8080/api/task/action", | ||
| "POST", | ||
| []byte(`{"taskID" : "0", "taskAction": "stop"}`), | ||
| }, | ||
| { | ||
| "http://127.0.0.1:8080/api/task/action", | ||
| "POST", | ||
| []byte(`{"taskAction": "stopAll"}`), | ||
| }, | ||
| { | ||
| "http://127.0.0.1:8080/api/task/action", | ||
| "POST", | ||
| []byte(`{"taskID" : "0", "taskAction": "query"}`), | ||
| }, | ||
| { | ||
| "http://127.0.0.1:8080/api/task/action", | ||
| "POST", | ||
| []byte(`{"taskAction": "queryAll"}`), | ||
| }, | ||
| } | ||
| for _, tc := range cases { | ||
| var Response Response | ||
| req, err := http.NewRequest(tc.requestMethod, tc.path, bytes.NewBuffer(tc.requestBody)) | ||
| req.Header.Set("Content-Type", "application/json") | ||
|
|
||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
|
|
||
| client := &http.Client{} | ||
| resp, err := client.Do(req) | ||
|
|
||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
|
|
||
| defer resp.Body.Close() | ||
|
|
||
| body, _ := ioutil.ReadAll(resp.Body) | ||
|
|
||
| json.Unmarshal([]byte(body), &Response) | ||
| if Response.Code != -1 && Response.Code != 0 { | ||
| t.Fail() | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.