feat: generate task per http range

generate task by for one file by http range

Signed-off-by: 慕陶 <[email protected]>

supernode/daemon/mgr/cdn/downloader.go

@@ -35,7 +35,7 @@ import (
 // Body which the caller is expected to close.
 func (cm *Manager) download(ctx context.Context, taskID, url string, headers map[string]string,
 	startPieceNum int, httpFileLength int64, pieceContSize int32) (*http.Response, error) {
-	var checkCode = http.StatusOK
+	var checkCode = http.StatusOK | http.StatusPartialContent
 
 	if startPieceNum > 0 {
 		breakRange, err := util.CalculateBreakRange(startPieceNum, int(pieceContSize), httpFileLength)
@@ -46,7 +46,10 @@ func (cm *Manager) download(ctx context.Context, taskID, url string, headers map
 		if headers == nil {
 			headers = make(map[string]string)
 		}
-		headers["Range"] = httputils.ConstructRangeStr(breakRange)
+		// check if Range in header? if Range already in Header, use this range directly
+		if _, ok := headers["Range"]; !ok {
+			headers["Range"] = httputils.ConstructRangeStr(breakRange)
+		}
 		checkCode = http.StatusPartialContent
 	}
 

supernode/daemon/mgr/task/manager_util.go

@@ -42,7 +42,7 @@ func (tm *Manager) addOrUpdateTask(ctx context.Context, req *types.TaskCreateReq
 	if stringutils.IsEmptyStr(req.TaskURL) {
 		taskURL = netutils.FilterURLParam(req.RawURL, req.Filter)
 	}
-	taskID := generateTaskID(taskURL, req.Md5, req.Identifier)
+	taskID := generateTaskID(taskURL, req.Md5, req.Identifier, req.Headers)
 
 	util.GetLock(taskID, true)
 	defer util.ReleaseLock(taskID, true)
@@ -480,15 +480,19 @@ func validateParams(req *types.TaskCreateRequest) error {
 
 // generateTaskID generates taskID with taskURL,md5 and identifier
 // and returns the SHA-256 checksum of the data.
-func generateTaskID(taskURL, md5, identifier string) string {
+func generateTaskID(taskURL, md5, identifier string, header map[string]string) string {
 	sign := ""
 	if !stringutils.IsEmptyStr(md5) {
 		sign = md5
 	} else if !stringutils.IsEmptyStr(identifier) {
 		sign = identifier
 	}
-	id := fmt.Sprintf("%s%s%s%s", key, taskURL, sign, key)
-
+	var id string
+	if r, ok := header["Range"]; ok {
+		id = fmt.Sprintf("%s%s%s%s%s", key, taskURL, sign, r, key)
+	} else {
+		id = fmt.Sprintf("%s%s%s%s", key, taskURL, sign, key)
+	}
 	return digest.Sha256(id)
 }
 
@@ -533,7 +537,7 @@ func (tm *Manager) getHTTPFileLength(taskID, url string, headers map[string]stri
 	if code == http.StatusUnauthorized || code == http.StatusProxyAuthRequired {
 		return -1, errors.Wrapf(errortypes.ErrAuthenticationRequired, "taskID: %s,code: %d", taskID, code)
 	}
-	if code != http.StatusOK {
+	if code != http.StatusOK && code != http.StatusPartialContent {
 		logrus.Warnf("failed to get http file length with unexpected code: %d", code)
 		if code == http.StatusNotFound {
 			return -1, errors.Wrapf(errortypes.ErrURLNotReachable, "taskID: %s, url: %s", taskID, url)

supernode/daemon/mgr/task/manager_util_test.go

@@ -73,7 +73,7 @@ func (s *TaskUtilTestSuite) TestEqualsTask(c *check.C) {
 	}{
 		{
 			existTask: &types.TaskInfo{
-				ID:             generateTaskID("http://aa.bb.com", "", ""),
+				ID:             generateTaskID("http://aa.bb.com", "", "", nil),
 				CdnStatus:      types.TaskInfoCdnStatusRUNNING,
 				HTTPFileLength: 1000,
 				PieceSize:      config.DefaultPieceSize,
@@ -83,7 +83,7 @@ func (s *TaskUtilTestSuite) TestEqualsTask(c *check.C) {
 				Md5:            "fooMD5",
 			},
 			task: &types.TaskInfo{
-				ID:             generateTaskID("http://aa.bb.com", "", ""),
+				ID:             generateTaskID("http://aa.bb.com", "", "", nil),
 				CdnStatus:      types.TaskInfoCdnStatusWAITING,
 				HTTPFileLength: 1000,
 				PieceSize:      config.DefaultPieceSize,
@@ -97,7 +97,7 @@ func (s *TaskUtilTestSuite) TestEqualsTask(c *check.C) {
 		{
 
 			existTask: &types.TaskInfo{
-				ID:             generateTaskID("http://aa.bb.com", "", ""),
+				ID:             generateTaskID("http://aa.bb.com", "", "", nil),
 				CdnStatus:      types.TaskInfoCdnStatusWAITING,
 				HTTPFileLength: 1000,
 				PieceSize:      config.DefaultPieceSize,
@@ -108,7 +108,7 @@ func (s *TaskUtilTestSuite) TestEqualsTask(c *check.C) {
 				Md5:            "fooMD5",
 			},
 			task: &types.TaskInfo{
-				ID:             generateTaskID("http://aa.bb.com", "", ""),
+				ID:             generateTaskID("http://aa.bb.com", "", "", nil),
 				CdnStatus:      types.TaskInfoCdnStatusWAITING,
 				HTTPFileLength: 1000,
 				PieceSize:      config.DefaultPieceSize,

supernode/httpclient/origin_http_client.go

@@ -163,7 +163,7 @@ func (client *OriginClient) Download(url string, headers map[string]string, chec
 		return nil, err
 	}
 
-	if resp.StatusCode == checkCode {
+	if (resp.StatusCode & checkCode) == resp.StatusCode {
 		return resp, nil
 	}
 	return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)