update IsValidURL

Signed-off-by: 楚贤 <[email protected]>

dfget/config/config.go

@@ -268,7 +268,7 @@ func AssertConfig(cfg *Config) (err error) {
 	}
 
 	if !netutils.IsValidURL(cfg.URL) {
-		return errors.Wrapf(errortypes.ErrInvalidValue, "url: %v", err)
+		return errors.Wrapf(errortypes.ErrInvalidValue, "url: %v", cfg.URL)
 	}
 
 	if err := checkOutput(cfg); err != nil {

dfget/config/config_test.go

@@ -101,7 +101,8 @@ func (suite *ConfigSuite) TestAssertConfig(c *check.C) {
 		checkFunc func(err error) bool
 	}{
 		{clog: clog, checkFunc: errortypes.IsInvalidValue},
-		{clog: clog, url: "http://a", checkFunc: errortypes.IsInvalidValue},
+		{clog: clog, url: "htt://a", checkFunc: errortypes.IsInvalidValue},
+		{clog: clog, url: "htt://a.b.com", checkFunc: errortypes.IsInvalidValue},
 		{clog: clog, url: "http://a.b.com", output: "/tmp/output", checkFunc: errortypes.IsNilError},
 		{clog: clog, url: "http://a.b.com", output: "/root", checkFunc: errortypes.IsInvalidValue},
 	}

pkg/httputils/http_util.go

@@ -55,8 +55,14 @@ const (
 // DefaultHTTPClient is the default implementation of SimpleHTTPClient.
 var DefaultHTTPClient SimpleHTTPClient = &defaultHTTPClient{}
 
+// protocols stores custom protocols
+// key: schema value: transport
 var protocols = sync.Map{}
 
+// validURLSchemas stores valid schemas
+// when call RegisterProtocol, validURLSchemas will be also updated.
+var validURLSchemas = "https?|HTTPS?"
+
 // SimpleHTTPClient defines some http functions used frequently.
 type SimpleHTTPClient interface {
 	PostJSON(url string, body interface{}, timeout time.Duration) (code int, res []byte, e error)
@@ -472,12 +478,14 @@ func handlePairRange(rangeStr string, length int64) (*RangeStruct, error) {
 	}, nil
 }
 
-// RegisterProtocol registers custom protocols in global variable "protocols" which will use in dfget and supernode
+// RegisterProtocol registers custom protocols in global variable "protocols" which will be used in dfget and supernode
 // Example:
 //   protocols := "helloworld"
 //   newTransport := funcNewTransport
 //   httputils.RegisterProtocol(protocols, newTransport)
+// RegisterProtocol must be called before initialise dfget or supernode instances.
 func RegisterProtocol(scheme string, rt http.RoundTripper) {
+	validURLSchemas += "|" + scheme
 	protocols.Store(scheme, rt)
 }
 
@@ -490,3 +498,7 @@ func RegisterProtocolOnTransport(tr *http.Transport) {
 			return true
 		})
 }
+
+func GetValidURLSchemas() string {
+	return validURLSchemas
+}

pkg/httputils/http_util_test.go

@@ -291,4 +291,4 @@ func (s *HTTPUtilTestSuite) TestRegisterProtocol(c *check.C) {
 
 	c.Assert(resp, check.NotNil)
 	c.Assert(resp.ContentLength, check.Equals, int64(-1))
-}
+}

pkg/netutils/netutils.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"regexp"
@@ -29,6 +30,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/dragonflyoss/Dragonfly/pkg/httputils"
 	"github.com/dragonflyoss/Dragonfly/pkg/rate"
 	"github.com/dragonflyoss/Dragonfly/pkg/stringutils"
 
@@ -201,13 +203,20 @@ func ConvertHeaders(headers []string) map[string]string {
 }
 
 // IsValidURL returns whether the string url is a valid HTTP URL.
-func IsValidURL(url string) bool {
-	// shorter than the shortest case 'http://a.b'
-	if len(url) < 10 {
+func IsValidURL(urlStr string) bool {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return false
+	}
+	if len(u.Host) == 0 || len(u.Scheme) == 0 {
 		return false
 	}
-	reg := regexp.MustCompile(`(https?|HTTPS?)://([\w_]+:[\w_]+@)?([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?`)
-	if result := reg.FindString(url); stringutils.IsEmptyStr(result) {
+
+	// with custom schemas, url like "x://y/z" is valid
+	reg := regexp.MustCompile(`(` +
+		httputils.GetValidURLSchemas() +
+		`)://([\w_]+:[\w_]+@)?([\w-]+\.)*[\w-]+(/[\w- ./?%&=]*)?`)
+	if result := reg.FindString(urlStr); stringutils.IsEmptyStr(result) {
 		return false
 	}
 	return true

pkg/netutils/netutils_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/dragonflyoss/Dragonfly/pkg/httputils"
 	"github.com/dragonflyoss/Dragonfly/pkg/rate"
 
 	"github.com/go-check/check"
@@ -127,11 +128,14 @@ func (suite *NetUtilSuite) TestFilterURLParam(c *check.C) {
 }
 
 func (suite *NetUtilSuite) TestIsValidURL(c *check.C) {
+	httputils.RegisterProtocol("test", nil)
+	httputils.RegisterProtocol("TEST", nil)
 	var cases = map[string]bool{
 		"":                      false,
-		"abcdefg":               false,
+		"abcdefg":               true,
 		"////a//":               false,
-		"a////a//":              false,
+		"a////a//":              true,
+		"a/b":                   true,
 		"a.com////a//":          true,
 		"a:[email protected]":             true,
 		"a:[email protected]":         true,
@@ -145,15 +149,19 @@ func (suite *NetUtilSuite) TestIsValidURL(c *check.C) {
 		"127.0.0.1:8080/我?x=1":  true,
 		"a.b":                   true,
 		"www.taobao.com":        true,
-		"http:/www.a.b.com":     false,
-		"https://github.com/dragonflyoss/Dragonfly/issues?" +
+		"github.com/dragonflyoss/Dragonfly/issues?" +
 			"q=is%3Aissue+is%3Aclosed": true,
+		// FIXME because x://y/z is valid, below urls is valid
+		//"http:/www.a.b.com":     false,
+		//"https://github.com/dragonflyoss/Dragonfly/issues?" +
+		//	"q=is%3Aissue+is%3Aclosed": false,
 	}
 
 	for k, v := range cases {
-		for _, scheme := range []string{"http", "https", "HTTP", "HTTPS"} {
+		for _, scheme := range []string{"http", "https", "HTTP", "HTTPS", "test", "TEST"} {
 			url := fmt.Sprintf("%s://%s", scheme, k)
 			result := IsValidURL(url)
+			c.Logf("%v %s", result, url)
 			c.Assert(result, check.Equals, v)
 		}
 	}