From d04b08c9b7ea3e20043ef4212ebe7609c1ce546f Mon Sep 17 00:00:00 2001 From: "allen.wq" Date: Wed, 13 May 2020 10:47:19 +0800 Subject: [PATCH] define the interface of downloader, data_scheduler, client_stream, register, uploader, data_protocol and url protocol. Signed-off-by: allen.wq --- dfget/corev2/basic/interface.go | 34 ++++++++ dfget/corev2/clientwriter/client_writer.go | 31 +++++++ dfget/corev2/datascheduler/data_scheduler.go | 65 +++++++++++++++ dfget/corev2/downloader/downloader.go | 28 +++++++ dfget/corev2/regist/register.go | 24 ++++++ dfget/corev2/report/reporter.go | 27 ++++++ dfget/corev2/uploader/uploader.go | 25 ++++++ pkg/protocol/distributiondata.go | 88 ++++++++++++++++++++ pkg/protocol/url.go | 74 ++++++++++++++++ 9 files changed, 396 insertions(+) create mode 100644 dfget/corev2/basic/interface.go create mode 100644 dfget/corev2/clientwriter/client_writer.go create mode 100644 dfget/corev2/datascheduler/data_scheduler.go create mode 100644 dfget/corev2/downloader/downloader.go create mode 100644 dfget/corev2/regist/register.go create mode 100644 dfget/corev2/report/reporter.go create mode 100644 dfget/corev2/uploader/uploader.go create mode 100644 pkg/protocol/distributiondata.go create mode 100644 pkg/protocol/url.go diff --git a/dfget/corev2/basic/interface.go b/dfget/corev2/basic/interface.go new file mode 100644 index 000000000..9493c32d3 --- /dev/null +++ b/dfget/corev2/basic/interface.go @@ -0,0 +1,34 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package basic + +// Response defines the response. +type Response interface { + Success() bool + Data() interface{} +} + +// RangeRequest defines the range request. +type RangeRequest interface { + URL() string + Offset() int64 + Size() int64 + Header() map[string]string + + // Extra gets the extra info. + Extra() interface{} +} diff --git a/dfget/corev2/clientwriter/client_writer.go b/dfget/corev2/clientwriter/client_writer.go new file mode 100644 index 000000000..2dcbfcc01 --- /dev/null +++ b/dfget/corev2/clientwriter/client_writer.go @@ -0,0 +1,31 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package clientwriter + +import ( + "github.com/dragonflyoss/Dragonfly/pkg/protocol" +) + +// ClientStream defines how to organize distribution data for range request. +// An instance binds to a range request. +// It may receive a lot of distribution data. +// Developer could add a io.WriteCloser in constructor of instance, and the ClientWriter will +// write request data to io.Writer. +type ClientWriter interface { + // WriteData writes the distribution data from other peers, it may be called more times. + PutData(data protocol.DistributionData) error +} diff --git a/dfget/corev2/datascheduler/data_scheduler.go b/dfget/corev2/datascheduler/data_scheduler.go new file mode 100644 index 000000000..951289c53 --- /dev/null +++ b/dfget/corev2/datascheduler/data_scheduler.go @@ -0,0 +1,65 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package datascheduler + +import ( + "context" + + "github.com/dragonflyoss/Dragonfly/dfget/corev2/basic" + + strfmt "github.com/go-openapi/strfmt" +) + +// PeerInfo represents the target address which is provided the download data. +type PeerInfo struct { + IP strfmt.IPv4 + Port int32 + Path string + // ID represents the client ID of peer. + ID string +} + +// SchedulerResult defines the result of schedule of range data. +type SchedulePieceDataResult struct { + Off int64 + Size int64 + + // PeerInfos represents the schedule peers which to get the range data. + PeerInfos []*PeerInfo +} + +// SchedulerResult defines the schedule result of request range. +// For some implementation, developer could do more than one schedule for the same request range. +type SchedulerResult interface { + // Result get the schedule result for range data which may not include all data of request range. + Result() []*SchedulePieceDataResult + + // State gets the temporary states of this schedule which binds to range request. + State() ScheduleState +} + +// ScheduleState defines the state of this schedule. +type ScheduleState interface { + // Continue tells user if reschedule the request range again. + Continue() bool +} + +// DataScheduler defines how to schedule peers for range request. +type DataScheduler interface { + // state should be got from SchedulerResult which is got from last caller for the same range request. + Schedule(ctx context.Context, rr basic.RangeRequest, state ScheduleState) (SchedulerResult, error) +} diff --git a/dfget/corev2/downloader/downloader.go b/dfget/corev2/downloader/downloader.go new file mode 100644 index 000000000..bb821dd6a --- /dev/null +++ b/dfget/corev2/downloader/downloader.go @@ -0,0 +1,28 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package downloader + +import ( + "context" + "io" +) + +// Downloader defines how to download file range from peer/cdn, an instance binds to . +type Downloader interface { + // Download downloads range data. + Download(ctx context.Context, off, size int64) (io.ReadCloser, error) +} diff --git a/dfget/corev2/regist/register.go b/dfget/corev2/regist/register.go new file mode 100644 index 000000000..f17aad7c1 --- /dev/null +++ b/dfget/corev2/regist/register.go @@ -0,0 +1,24 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package regist + +import "github.com/dragonflyoss/Dragonfly/dfget/corev2/basic" + +// SupernodeRegister encapsulates the Register steps into a struct. +type SupernodeRegister interface { + Register(peerPort int) (response basic.Response, err error) +} diff --git a/dfget/corev2/report/reporter.go b/dfget/corev2/report/reporter.go new file mode 100644 index 000000000..5be4ae199 --- /dev/null +++ b/dfget/corev2/report/reporter.go @@ -0,0 +1,27 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package report + +import ( + "github.com/dragonflyoss/Dragonfly/dfget/corev2/basic" + "github.com/dragonflyoss/Dragonfly/dfget/locator" +) + +// Reporter defines how to report resource to suprnode. +type Reporter interface { + Report(supernode *locator.Supernode) (basic.Response, error) +} diff --git a/dfget/corev2/uploader/uploader.go b/dfget/corev2/uploader/uploader.go new file mode 100644 index 000000000..e55eb0f90 --- /dev/null +++ b/dfget/corev2/uploader/uploader.go @@ -0,0 +1,25 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package uploader + +import "io" + +// Uploader defines how to upload range by path. +type Uploader interface { + // UploadRange defines how to upload range by path. + UploadRange(path string, off, size int64, opt interface{}) (io.ReadCloser, error) +} diff --git a/pkg/protocol/distributiondata.go b/pkg/protocol/distributiondata.go new file mode 100644 index 000000000..ab78d6ea9 --- /dev/null +++ b/pkg/protocol/distributiondata.go @@ -0,0 +1,88 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protocol + +import ( + "context" + "io" +) + +// DataEncoder defines how to encode/decode data. +type DataEncoder interface { + // Encode data. + Encode(io.Reader) (io.Reader, error) + + // Decode data. + Decode(io.Reader) (io.Reader, error) +} + +// DataType defines the type of DistributionData. +type DataType interface { + // String return the type string. + String() string + + // Encoder return the encoder of the type. + Encoder() DataEncoder +} + +// DistributionData defines the protocol of distribute data which is exchanged in peers. +type DistributionData interface { + // Type gets the data type. + Type() DataType + + // Size gets the size of data. + Size() int64 + + // Metadata gets the metadata. + Metadata() interface{} + + // Content gets the content of data. + Content(ctx context.Context) (io.Reader, error) +} + +type eofDataType struct{} + +func (ty eofDataType) String() string { + return "EOF" +} + +func (ty *eofDataType) Encoder() DataEncoder { + return nil +} + +// EOFDistributionData represents the eof of file. +type eofDistributionData struct{} + +func (eof *eofDistributionData) Size() int64 { + return 0 +} + +func (eof *eofDistributionData) Metadata() interface{} { + return nil +} + +func (eof *eofDistributionData) Content(ctx context.Context) (io.Reader, error) { + return nil, io.EOF +} + +func (eof *eofDistributionData) Type() DataType { + return &eofDataType{} +} + +func NewEoFDistributionData() DistributionData { + return &eofDistributionData{} +} diff --git a/pkg/protocol/url.go b/pkg/protocol/url.go new file mode 100644 index 000000000..0aeeb4b8e --- /dev/null +++ b/pkg/protocol/url.go @@ -0,0 +1,74 @@ +/* + * Copyright The Dragonfly Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package protocol + +import ( + "context" + "io" +) + +// Metadata defines how to operate the metadata. +type Metadata interface { + Get(key string) (interface{}, error) + Set(key string, value interface{}) + Del(key string) + All() interface{} +} + +// Resource defines the way how to get some information from remote resource. +// An instance should bind a resource. +// Developers can implement their own Resource which could support different protocol. +type Resource interface { + // Read gets range data from the binding resource. + Read(ctx context.Context, off int64, size int64) (io.ReadCloser, error) + + // Length gets the length of binding resource. + Length(ctx context.Context) (int64, error) + + // Metadata gets the metadata of binding resource. + Metadata(ctx context.Context) (Metadata, error) + + // Expire gets if the binding resource is expired. + Expire(ctx context.Context) (bool, interface{}, error) + + // Call allows user defined request. + Call(ctx context.Context, request interface{}) (response interface{}, err error) + + // Close the resource. + io.Closer +} + +// Client defines how to get resource. +type Client interface { + // GetResource gets resource by argument. + GetResource(url string, md Metadata) Resource +} + +// ClientBuilder defines how to create an instance of Client. +type ClientBuilder interface { + // NewProtocolClient creates an instance of Client. + NewProtocolClient(clientOpt interface{}) (Client, error) +} + +// ClientRegister defines how to register pair . +type ClientRegister interface { + // ClientRegister registers pair . + RegisterProtocol(protocol string, builder ClientBuilder) + + // GetClientBuilder gets the ClientBuilder by protocol. + GetClientBuilder(protocol string) (ClientBuilder, error) +}