From 768851b3291e4dcdcb6c66af1e1641bca5c6c1d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A5=9A=E8=B4=A4?= Date: Fri, 13 Mar 2020 10:38:46 +0800 Subject: [PATCH] update IsValidURL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 楚贤 --- dfget/config/config.go | 2 +- dfget/config/config_test.go | 3 ++- pkg/httputils/http_util.go | 14 +++++++++++++- pkg/httputils/http_util_test.go | 2 +- pkg/netutils/netutils.go | 19 ++++++++++++++----- pkg/netutils/netutils_test.go | 18 +++++++++++++----- 6 files changed, 44 insertions(+), 14 deletions(-) diff --git a/dfget/config/config.go b/dfget/config/config.go index be1b45f12..cc2344e58 100644 --- a/dfget/config/config.go +++ b/dfget/config/config.go @@ -268,7 +268,7 @@ func AssertConfig(cfg *Config) (err error) { } if !netutils.IsValidURL(cfg.URL) { - return errors.Wrapf(errortypes.ErrInvalidValue, "url: %v", err) + return errors.Wrapf(errortypes.ErrInvalidValue, "url: %v", cfg.URL) } if err := checkOutput(cfg); err != nil { diff --git a/dfget/config/config_test.go b/dfget/config/config_test.go index cc39d2391..ff82c905e 100644 --- a/dfget/config/config_test.go +++ b/dfget/config/config_test.go @@ -101,7 +101,8 @@ func (suite *ConfigSuite) TestAssertConfig(c *check.C) { checkFunc func(err error) bool }{ {clog: clog, checkFunc: errortypes.IsInvalidValue}, - {clog: clog, url: "http://a", checkFunc: errortypes.IsInvalidValue}, + {clog: clog, url: "htt://a", checkFunc: errortypes.IsInvalidValue}, + {clog: clog, url: "htt://a.b.com", checkFunc: errortypes.IsInvalidValue}, {clog: clog, url: "http://a.b.com", output: "/tmp/output", checkFunc: errortypes.IsNilError}, {clog: clog, url: "http://a.b.com", output: "/root", checkFunc: errortypes.IsInvalidValue}, } diff --git a/pkg/httputils/http_util.go b/pkg/httputils/http_util.go index 8cfada10b..583ba677f 100644 --- a/pkg/httputils/http_util.go +++ b/pkg/httputils/http_util.go @@ -55,8 +55,14 @@ const ( // DefaultHTTPClient is the default implementation of SimpleHTTPClient. var DefaultHTTPClient SimpleHTTPClient = &defaultHTTPClient{} +// protocols stores custom protocols +// key: schema value: transport var protocols = sync.Map{} +// validURLSchemas stores valid schemas +// when call RegisterProtocol, validURLSchemas will be also updated. +var validURLSchemas = "https?|HTTPS?" + // SimpleHTTPClient defines some http functions used frequently. type SimpleHTTPClient interface { PostJSON(url string, body interface{}, timeout time.Duration) (code int, res []byte, e error) @@ -472,12 +478,14 @@ func handlePairRange(rangeStr string, length int64) (*RangeStruct, error) { }, nil } -// RegisterProtocol registers custom protocols in global variable "protocols" which will use in dfget and supernode +// RegisterProtocol registers custom protocols in global variable "protocols" which will be used in dfget and supernode // Example: // protocols := "helloworld" // newTransport := funcNewTransport // httputils.RegisterProtocol(protocols, newTransport) +// RegisterProtocol must be called before initialise dfget or supernode instances. func RegisterProtocol(scheme string, rt http.RoundTripper) { + validURLSchemas += "|" + scheme protocols.Store(scheme, rt) } @@ -490,3 +498,7 @@ func RegisterProtocolOnTransport(tr *http.Transport) { return true }) } + +func GetValidURLSchemas() string { + return validURLSchemas +} diff --git a/pkg/httputils/http_util_test.go b/pkg/httputils/http_util_test.go index 6f7213964..ce5727ff4 100644 --- a/pkg/httputils/http_util_test.go +++ b/pkg/httputils/http_util_test.go @@ -291,4 +291,4 @@ func (s *HTTPUtilTestSuite) TestRegisterProtocol(c *check.C) { c.Assert(resp, check.NotNil) c.Assert(resp.ContentLength, check.Equals, int64(-1)) -} \ No newline at end of file +} diff --git a/pkg/netutils/netutils.go b/pkg/netutils/netutils.go index f90aff8a3..098935225 100644 --- a/pkg/netutils/netutils.go +++ b/pkg/netutils/netutils.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "net/http" + "net/url" "os" "os/exec" "regexp" @@ -29,6 +30,7 @@ import ( "strings" "time" + "github.com/dragonflyoss/Dragonfly/pkg/httputils" "github.com/dragonflyoss/Dragonfly/pkg/rate" "github.com/dragonflyoss/Dragonfly/pkg/stringutils" @@ -201,13 +203,20 @@ func ConvertHeaders(headers []string) map[string]string { } // IsValidURL returns whether the string url is a valid HTTP URL. -func IsValidURL(url string) bool { - // shorter than the shortest case 'http://a.b' - if len(url) < 10 { +func IsValidURL(urlStr string) bool { + u, err := url.Parse(urlStr) + if err != nil { + return false + } + if len(u.Host) == 0 || len(u.Scheme) == 0 { return false } - reg := regexp.MustCompile(`(https?|HTTPS?)://([\w_]+:[\w_]+@)?([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?`) - if result := reg.FindString(url); stringutils.IsEmptyStr(result) { + + // with custom schemas, url like "x://y/z" is valid + reg := regexp.MustCompile(`(` + + httputils.GetValidURLSchemas() + + `)://([\w_]+:[\w_]+@)?([\w-]+\.)*[\w-]+(/[\w- ./?%&=]*)?`) + if result := reg.FindString(urlStr); stringutils.IsEmptyStr(result) { return false } return true diff --git a/pkg/netutils/netutils_test.go b/pkg/netutils/netutils_test.go index cc538c3bc..b7eda3589 100644 --- a/pkg/netutils/netutils_test.go +++ b/pkg/netutils/netutils_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/dragonflyoss/Dragonfly/pkg/httputils" "github.com/dragonflyoss/Dragonfly/pkg/rate" "github.com/go-check/check" @@ -127,11 +128,14 @@ func (suite *NetUtilSuite) TestFilterURLParam(c *check.C) { } func (suite *NetUtilSuite) TestIsValidURL(c *check.C) { + httputils.RegisterProtocol("test", nil) + httputils.RegisterProtocol("TEST", nil) var cases = map[string]bool{ "": false, - "abcdefg": false, + "abcdefg": true, "////a//": false, - "a////a//": false, + "a////a//": true, + "a/b": true, "a.com////a//": true, "a:b@a.com": true, "a:b@127.0.0.1": true, @@ -145,15 +149,19 @@ func (suite *NetUtilSuite) TestIsValidURL(c *check.C) { "127.0.0.1:8080/我?x=1": true, "a.b": true, "www.taobao.com": true, - "http:/www.a.b.com": false, - "https://github.com/dragonflyoss/Dragonfly/issues?" + + "github.com/dragonflyoss/Dragonfly/issues?" + "q=is%3Aissue+is%3Aclosed": true, + // FIXME because x://y/z is valid, below urls is valid + //"http:/www.a.b.com": false, + //"https://github.com/dragonflyoss/Dragonfly/issues?" + + // "q=is%3Aissue+is%3Aclosed": false, } for k, v := range cases { - for _, scheme := range []string{"http", "https", "HTTP", "HTTPS"} { + for _, scheme := range []string{"http", "https", "HTTP", "HTTPS", "test", "TEST"} { url := fmt.Sprintf("%s://%s", scheme, k) result := IsValidURL(url) + c.Logf("%v %s", result, url) c.Assert(result, check.Equals, v) } }