From 99918a712d951fdf6d49ae762a118e7db8674c35 Mon Sep 17 00:00:00 2001 From: lowzj Date: Wed, 11 Mar 2020 10:56:47 +0800 Subject: [PATCH] feature: define the interface and http api of preheat Signed-off-by: lowzj --- apis/swagger.yml | 49 +++++++- apis/types/preheat_create_request.go | 81 ++++++++++++- apis/types/preheat_info.go | 46 +------ apis/types/preheat_status.go | 75 ++++++++++++ client/preheat_info_test.go | 4 +- supernode/daemon/mgr/preheat/manager.go | 52 ++++++++ supernode/daemon/mgr/preheat_mgr.go | 61 ++++++++++ supernode/server/preheat_bridge.go | 155 ++++++++++++++++++++++++ supernode/server/router.go | 2 +- supernode/server/server.go | 8 ++ 10 files changed, 483 insertions(+), 50 deletions(-) create mode 100644 apis/types/preheat_status.go create mode 100644 supernode/daemon/mgr/preheat/manager.go create mode 100644 supernode/daemon/mgr/preheat_mgr.go create mode 100644 supernode/server/preheat_bridge.go diff --git a/apis/swagger.yml b/apis/swagger.yml index e0fd68d3b..f21da9a4d 100644 --- a/apis/swagger.yml +++ b/apis/swagger.yml @@ -580,6 +580,10 @@ paths: description: "bad parameter" schema: $ref: '#/definitions/Error' + 409: + description: "preheat task already exists" + schema: + $ref: '#/definitions/Error' 500: $ref: "#/responses/500ErrorResponse" @@ -628,6 +632,28 @@ paths: 500: $ref: "#/responses/500ErrorResponse" + delete: + summary: "Delete a preheat task" + description: | + delete a preheat task + produces: + - "application/json" + parameters: + - name: id + in: path + required: true + description: "ID of preheat task" + type: string + responses: + 200: + description: "no error" + 404: + description: "no such preheat task" + schema: + $ref: "#/responses/404ErrorResponse" + 500: + $ref: "#/responses/500ErrorResponse" + /task/metrics: post: summary: "upload dfclient download metrics" @@ -1225,7 +1251,7 @@ definitions: taskId: type: "string" description: | - the taskID of the piece. + the taskID of the piece. srcCid: type: "string" description: | @@ -1270,7 +1296,7 @@ definitions: description: | ID of preheat task. status: - type: "string" + $ref: "#/definitions/PreheatStatus" description: | The status of preheat task. WAITING -----> RUNNING -----> SUCCESS @@ -1278,7 +1304,6 @@ definitions: The initial status of a created preheat task is WAITING. It's finished when a preheat task's status is FAILED or SUCCESS. A finished preheat task's information can be queried within 24 hours. - enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS"] startTime: type: "string" format: "date-time" @@ -1288,17 +1313,33 @@ definitions: format: "date-time" description: "the preheat task finish time" + PreheatStatus: + type: string + description: | + The status of preheat task. + WAITING -----> RUNNING -----> SUCCESS + |--> FAILED + The initial status of a created preheat task is WAITING. + It's finished when a preheat task's status is FAILED or SUCCESS. + A finished preheat task's information can be queried within 24 hours. + enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS"] + PreheatCreateRequest: type: "object" description: | Request option of creating a preheat task in supernode. + required: + - type + - url properties: type: type: "string" + enum: ["image", "file"] description: | this must be image or file url: type: "string" + minLength: 3 description: "the image or file location" filter: type: "string" @@ -1356,7 +1397,7 @@ definitions: status: type: "string" description: | - The status of Dfget download process. + The status of Dfget download process. enum: ["WAITING", "RUNNING", "FAILED", "SUCCESS"] peerID: type: "string" diff --git a/apis/types/preheat_create_request.go b/apis/types/preheat_create_request.go index eecee2c47..2a49a383f 100644 --- a/apis/types/preheat_create_request.go +++ b/apis/types/preheat_create_request.go @@ -6,9 +6,13 @@ package types // Editing this file might prove futile when you re-run the swagger generate command import ( + "encoding/json" + strfmt "github.com/go-openapi/strfmt" + "github.com/go-openapi/errors" "github.com/go-openapi/swag" + "github.com/go-openapi/validate" ) // PreheatCreateRequest Request option of creating a preheat task in supernode. @@ -33,14 +37,87 @@ type PreheatCreateRequest struct { // this must be image or file // - Type string `json:"type,omitempty"` + // Required: true + // Enum: [image file] + Type *string `json:"type"` // the image or file location - URL string `json:"url,omitempty"` + // Required: true + // Min Length: 3 + URL *string `json:"url"` } // Validate validates this preheat create request func (m *PreheatCreateRequest) Validate(formats strfmt.Registry) error { + var res []error + + if err := m.validateType(formats); err != nil { + res = append(res, err) + } + + if err := m.validateURL(formats); err != nil { + res = append(res, err) + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +var preheatCreateRequestTypeTypePropEnum []interface{} + +func init() { + var res []string + if err := json.Unmarshal([]byte(`["image","file"]`), &res); err != nil { + panic(err) + } + for _, v := range res { + preheatCreateRequestTypeTypePropEnum = append(preheatCreateRequestTypeTypePropEnum, v) + } +} + +const ( + + // PreheatCreateRequestTypeImage captures enum value "image" + PreheatCreateRequestTypeImage string = "image" + + // PreheatCreateRequestTypeFile captures enum value "file" + PreheatCreateRequestTypeFile string = "file" +) + +// prop value enum +func (m *PreheatCreateRequest) validateTypeEnum(path, location string, value string) error { + if err := validate.Enum(path, location, value, preheatCreateRequestTypeTypePropEnum); err != nil { + return err + } + return nil +} + +func (m *PreheatCreateRequest) validateType(formats strfmt.Registry) error { + + if err := validate.Required("type", "body", m.Type); err != nil { + return err + } + + // value enum + if err := m.validateTypeEnum("type", "body", *m.Type); err != nil { + return err + } + + return nil +} + +func (m *PreheatCreateRequest) validateURL(formats strfmt.Registry) error { + + if err := validate.Required("url", "body", m.URL); err != nil { + return err + } + + if err := validate.MinLength("url", "body", string(*m.URL), 3); err != nil { + return err + } + return nil } diff --git a/apis/types/preheat_info.go b/apis/types/preheat_info.go index 31664e307..2993a500e 100644 --- a/apis/types/preheat_info.go +++ b/apis/types/preheat_info.go @@ -6,8 +6,6 @@ package types // Editing this file might prove futile when you re-run the swagger generate command import ( - "encoding/json" - strfmt "github.com/go-openapi/strfmt" "github.com/go-openapi/errors" @@ -40,8 +38,7 @@ type PreheatInfo struct { // It's finished when a preheat task's status is FAILED or SUCCESS. // A finished preheat task's information can be queried within 24 hours. // - // Enum: [WAITING RUNNING FAILED SUCCESS] - Status string `json:"status,omitempty"` + Status PreheatStatus `json:"status,omitempty"` } // Validate validates this preheat info @@ -92,49 +89,16 @@ func (m *PreheatInfo) validateStartTime(formats strfmt.Registry) error { return nil } -var preheatInfoTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["WAITING","RUNNING","FAILED","SUCCESS"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - preheatInfoTypeStatusPropEnum = append(preheatInfoTypeStatusPropEnum, v) - } -} - -const ( - - // PreheatInfoStatusWAITING captures enum value "WAITING" - PreheatInfoStatusWAITING string = "WAITING" - - // PreheatInfoStatusRUNNING captures enum value "RUNNING" - PreheatInfoStatusRUNNING string = "RUNNING" - - // PreheatInfoStatusFAILED captures enum value "FAILED" - PreheatInfoStatusFAILED string = "FAILED" - - // PreheatInfoStatusSUCCESS captures enum value "SUCCESS" - PreheatInfoStatusSUCCESS string = "SUCCESS" -) - -// prop value enum -func (m *PreheatInfo) validateStatusEnum(path, location string, value string) error { - if err := validate.Enum(path, location, value, preheatInfoTypeStatusPropEnum); err != nil { - return err - } - return nil -} - func (m *PreheatInfo) validateStatus(formats strfmt.Registry) error { if swag.IsZero(m.Status) { // not required return nil } - // value enum - if err := m.validateStatusEnum("status", "body", m.Status); err != nil { + if err := m.Status.Validate(formats); err != nil { + if ve, ok := err.(*errors.Validation); ok { + return ve.ValidateName("status") + } return err } diff --git a/apis/types/preheat_status.go b/apis/types/preheat_status.go new file mode 100644 index 000000000..caf562d42 --- /dev/null +++ b/apis/types/preheat_status.go @@ -0,0 +1,75 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package types + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "encoding/json" + + strfmt "github.com/go-openapi/strfmt" + + "github.com/go-openapi/errors" + "github.com/go-openapi/validate" +) + +// PreheatStatus The status of preheat task. +// WAITING -----> RUNNING -----> SUCCESS +// |--> FAILED +// The initial status of a created preheat task is WAITING. +// It's finished when a preheat task's status is FAILED or SUCCESS. +// A finished preheat task's information can be queried within 24 hours. +// +// swagger:model PreheatStatus +type PreheatStatus string + +const ( + + // PreheatStatusWAITING captures enum value "WAITING" + PreheatStatusWAITING PreheatStatus = "WAITING" + + // PreheatStatusRUNNING captures enum value "RUNNING" + PreheatStatusRUNNING PreheatStatus = "RUNNING" + + // PreheatStatusFAILED captures enum value "FAILED" + PreheatStatusFAILED PreheatStatus = "FAILED" + + // PreheatStatusSUCCESS captures enum value "SUCCESS" + PreheatStatusSUCCESS PreheatStatus = "SUCCESS" +) + +// for schema +var preheatStatusEnum []interface{} + +func init() { + var res []PreheatStatus + if err := json.Unmarshal([]byte(`["WAITING","RUNNING","FAILED","SUCCESS"]`), &res); err != nil { + panic(err) + } + for _, v := range res { + preheatStatusEnum = append(preheatStatusEnum, v) + } +} + +func (m PreheatStatus) validatePreheatStatusEnum(path, location string, value PreheatStatus) error { + if err := validate.Enum(path, location, value, preheatStatusEnum); err != nil { + return err + } + return nil +} + +// Validate validates this preheat status +func (m PreheatStatus) Validate(formats strfmt.Registry) error { + var res []error + + // value enum + if err := m.validatePreheatStatusEnum("", "body", m); err != nil { + return err + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/client/preheat_info_test.go b/client/preheat_info_test.go index 028893802..f6c32c946 100644 --- a/client/preheat_info_test.go +++ b/client/preheat_info_test.go @@ -53,7 +53,7 @@ func TestPreheatInfo(t *testing.T) { id := "1234567890" startTime := strfmt.DateTime(time.Now()) finishTime := strfmt.DateTime(time.Now().Add(time.Duration(time.Minute))) - status := "SUCCESS" + status := types.PreheatStatusSUCCESS expectedURL := fmt.Sprintf("/preheats/%s", id) @@ -65,7 +65,7 @@ func TestPreheatInfo(t *testing.T) { ID: id, StartTime: startTime, FinishTime: finishTime, - Status: "SUCCESS", + Status: types.PreheatStatusSUCCESS, } b, err := json.Marshal(info) if err != nil { diff --git a/supernode/daemon/mgr/preheat/manager.go b/supernode/daemon/mgr/preheat/manager.go new file mode 100644 index 000000000..2f9f6e528 --- /dev/null +++ b/supernode/daemon/mgr/preheat/manager.go @@ -0,0 +1,52 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package preheat + +import ( + "context" + "fmt" + + "github.com/dragonflyoss/Dragonfly/apis/types" + "github.com/dragonflyoss/Dragonfly/supernode/config" + "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr" +) + +var _ mgr.PreheatManager = &Manager{} + +// Manager is an implementation of interface PreheatManager. +type Manager struct { +} + +func NewManager(cfg *config.Config) (mgr.PreheatManager, error) { + return &Manager{}, nil +} + +func (m *Manager) Create(ctx context.Context, task *types.PreheatCreateRequest) (preheatID string, err error) { + return "", fmt.Errorf("not implement") +} + +func (m *Manager) Get(ctx context.Context, preheatID string) (preheatTask *mgr.PreheatTask, err error) { + return nil, fmt.Errorf("not implement") +} + +func (m *Manager) Delete(ctx context.Context, preheatID string) (err error) { + return fmt.Errorf("not implement") +} + +func (m *Manager) GetAll(ctx context.Context) (preheatTasks []*mgr.PreheatTask, err error) { + return nil, fmt.Errorf("not implement") +} diff --git a/supernode/daemon/mgr/preheat_mgr.go b/supernode/daemon/mgr/preheat_mgr.go new file mode 100644 index 000000000..cc5d9194a --- /dev/null +++ b/supernode/daemon/mgr/preheat_mgr.go @@ -0,0 +1,61 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mgr + +import ( + "context" + + "github.com/dragonflyoss/Dragonfly/apis/types" +) + +// PreheatTask stores the detailed preheat task information. +type PreheatTask struct { + ID string + URL string + Type string + Filter string + Identifier string + Headers map[string]string + + // ParentID records its parent preheat task id. Sometimes the current + // preheat task is not created by user directly. Such as preheating an + // image, it contains several layers that should be preheated together. + // So the image preheat task is the parent of its layer preheat tasks. + ParentID string + Children []string + + Status types.PreheatStatus + StartTime int64 + FinishTime int64 + ErrorMsg string +} + +// PreheatManager provides basic operations of preheat. +type PreheatManager interface { + // Create creates a preheat task to cache data in supernode, thus accelerating the + // p2p downloading. + Create(ctx context.Context, task *types.PreheatCreateRequest) (preheatID string, err error) + + // Get gets detailed preheat task information by preheatID. + Get(ctx context.Context, preheatID string) (preheatTask *PreheatTask, err error) + + // Delete deletes a preheat task by preheatID. + Delete(ctx context.Context, preheatID string) (err error) + + // GetAll gets all preheat tasks that unexpired. + GetAll(ctx context.Context) (preheatTask []*PreheatTask, err error) +} diff --git a/supernode/server/preheat_bridge.go b/supernode/server/preheat_bridge.go new file mode 100644 index 000000000..f3e302d9e --- /dev/null +++ b/supernode/server/preheat_bridge.go @@ -0,0 +1,155 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "context" + "encoding/json" + "io" + "net/http" + + "github.com/dragonflyoss/Dragonfly/apis/types" + "github.com/dragonflyoss/Dragonfly/pkg/errortypes" + + "github.com/go-openapi/strfmt" + "github.com/gorilla/mux" + "github.com/sirupsen/logrus" +) + +// --------------------------------------------------------------------------- +// handlers of preheat http apis + +func (s *Server) createPreheatTask(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { + request := &types.PreheatCreateRequest{} + if err := parseRequest(req.Body, request, request.Validate); err != nil { + return err + } + preheatID, err := s.PreheatMgr.Create(ctx, request) + if err != nil { + return err + } + resp := types.PreheatCreateResponse{ID: preheatID} + return EncodeResponse(rw, http.StatusCreated, resp) +} + +func (s *Server) getAllPreheatTasks(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { + tasks, err := s.PreheatMgr.GetAll(ctx) + if err != nil { + return err + } + return EncodeResponse(rw, http.StatusOK, tasks) +} + +func (s *Server) getPreheatTask(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { + id := mux.Vars(req)["id"] + task, err := s.PreheatMgr.Get(ctx, id) + if err != nil { + return err + } + resp := types.PreheatInfo{ + ID: task.ID, + FinishTime: strfmt.NewDateTime(), + StartTime: strfmt.NewDateTime(), + Status: task.Status, + } + return EncodeResponse(rw, http.StatusOK, resp) +} + +func (s *Server) deletePreheatTask(ctx context.Context, rw http.ResponseWriter, req *http.Request) error { + id := mux.Vars(req)["id"] + if err := s.PreheatMgr.Delete(ctx, id); err != nil { + return err + } + return EncodeResponse(rw, http.StatusOK, true) +} + +// --------------------------------------------------------------------------- +// helper functions + +type validateFunc func(registry strfmt.Registry) error + +func parseRequest(body io.Reader, request interface{}, validator validateFunc) error { + if err := json.NewDecoder(body).Decode(request); err != nil { + if err == io.EOF { + return errortypes.New(http.StatusBadRequest, "empty body") + } + return errortypes.New(http.StatusBadRequest, err.Error()) + } + if validator != nil { + if err := validator(strfmt.NewFormats()); err != nil { + return errortypes.New(http.StatusBadRequest, err.Error()) + } + } + return nil +} + +// initPreheatHandlers register preheat apis +func initPreheatHandlers(s *Server, r *mux.Router) { + handlers := []*HandlerSpec{ + {Method: http.MethodPost, Path: "/preheats", HandlerFunc: s.createPreheatTask}, + {Method: http.MethodGet, Path: "/preheats", HandlerFunc: s.getAllPreheatTasks}, + {Method: http.MethodGet, Path: "/preheats/{id}", HandlerFunc: s.getPreheatTask}, + {Method: http.MethodDelete, Path: "/preheats/{id}", HandlerFunc: s.deletePreheatTask}, + } + // register API + for _, h := range handlers { + if h != nil { + r.Path(versionMatcher + h.Path).Methods(h.Method). + Handler(m.instrumentHandler(h.Path, postPreheatHandler(h.HandlerFunc))) + r.Path("/api/v1" + h.Path).Methods(h.Method). + Handler(m.instrumentHandler(h.Path, postPreheatHandler(h.HandlerFunc))) + r.Path(h.Path).Methods(h.Method). + Handler(m.instrumentHandler(h.Path, postPreheatHandler(h.HandlerFunc))) + } + } +} + +func postPreheatHandler(h Handler) http.HandlerFunc { + pctx := context.Background() + + return func(w http.ResponseWriter, req *http.Request) { + ctx, cancel := context.WithCancel(pctx) + defer cancel() + + // Start to handle request. + err := h(ctx, w, req) + if err != nil { + // Handle error if request handling fails. + handlePreheatErrorResponse(w, err) + } + logrus.Debugf("%s %v err:%v", req.Method, req.URL, err) + } +} + +func handlePreheatErrorResponse(w http.ResponseWriter, err error) { + var ( + code int + errMsg string + ) + + // By default, daemon side returns code 500 if error happens. + code = http.StatusInternalServerError + if e, ok := err.(*errortypes.DfError); ok { + code = e.Code + errMsg = e.Msg + } + + _ = EncodeResponse(w, code, types.ErrorResponse{ + Code: int64(code), + Message: errMsg, + }) +} diff --git a/supernode/server/router.go b/supernode/server/router.go index cb4121135..896f10c40 100644 --- a/supernode/server/router.go +++ b/supernode/server/router.go @@ -24,7 +24,6 @@ import ( "github.com/dragonflyoss/Dragonfly/apis/types" "github.com/dragonflyoss/Dragonfly/version" - "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -75,6 +74,7 @@ func initRoute(s *Server) *mux.Router { r.Path(h.Path).Methods(h.Method).Handler(m.instrumentHandler(h.Path, filter(h.HandlerFunc))) } } + initPreheatHandlers(s, r) if s.Config.Debug || s.Config.EnableProfiler { r.PathPrefix("/debug/pprof/cmdline").HandlerFunc(pprof.Cmdline) diff --git a/supernode/server/server.go b/supernode/server/server.go index fcc5d4129..ed89372a9 100644 --- a/supernode/server/server.go +++ b/supernode/server/server.go @@ -29,6 +29,7 @@ import ( "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/gc" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/peer" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/pieceerror" + "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/preheat" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/progress" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/scheduler" "github.com/dragonflyoss/Dragonfly/supernode/daemon/mgr/task" @@ -51,6 +52,7 @@ type Server struct { ProgressMgr mgr.ProgressMgr GCMgr mgr.GCMgr PieceErrorMgr mgr.PieceErrorMgr + PreheatMgr mgr.PreheatManager originClient httpclient.OriginHTTPClient } @@ -114,6 +116,11 @@ func New(cfg *config.Config, logger *logrus.Logger, register prometheus.Register return nil, err } + preheatMgr, err := preheat.NewManager(cfg) + if err != nil { + return nil, err + } + return &Server{ Config: cfg, PeerMgr: peerMgr, @@ -122,6 +129,7 @@ func New(cfg *config.Config, logger *logrus.Logger, register prometheus.Register ProgressMgr: progressMgr, GCMgr: gcMgr, PieceErrorMgr: pieceErrorMgr, + PreheatMgr: preheatMgr, originClient: originClient, }, nil }