From e135cc7095cacccc5bf1258b9c7e18f238a4d3c6 Mon Sep 17 00:00:00 2001 From: Yota Toyama Date: Wed, 11 Nov 2020 12:55:31 -0800 Subject: [PATCH] HTTP proxy support (#132) * Parse proxy argument * Use proxy * Change test proxy url * Add value name * Bump version * Update help message * Update help message again --- .snapshots/TestHelp | 1 + arguments.go | 1 + arguments_test.go | 1 + command.go | 1 + configuration.go | 2 +- fasthttp_http_client_factory.go | 13 ++++++++++--- go.sum | 18 ++++-------------- http_client_factory.go | 1 + 8 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.snapshots/TestHelp b/.snapshots/TestHelp index 27ae6a6e..669b017d 100644 --- a/.snapshots/TestHelp +++ b/.snapshots/TestHelp @@ -22,6 +22,7 @@ Application Options: -t, --timeout= Timeout for HTTP requests in seconds (default: 10) -v, --verbose Show successful results too + --proxy= HTTP proxy host --skip-tls-verification Skip TLS certificate verification --one-page-only Only check links found in the given URL diff --git a/arguments.go b/arguments.go index 08b7751b..32d87877 100644 --- a/arguments.go +++ b/arguments.go @@ -22,6 +22,7 @@ type arguments struct { MaxRedirections int `short:"r" long:"max-redirections" value-name:"" default:"64" description:"Maximum number of redirections"` Timeout int `short:"t" long:"timeout" value-name:"" default:"10" description:"Timeout for HTTP requests in seconds"` Verbose bool `short:"v" long:"verbose" description:"Show successful results too"` + Proxy string `long:"proxy" value-name:"" description:"HTTP proxy host"` SkipTLSVerification bool `long:"skip-tls-verification" description:"Skip TLS certificate verification"` OnePageOnly bool `long:"one-page-only" description:"Only check links found in the given URL"` Color color `long:"color" description:"Color output" choice:"auto" choice:"always" choice:"never" default:"auto"` diff --git a/arguments_test.go b/arguments_test.go index 778b34cf..77e6d42f 100644 --- a/arguments_test.go +++ b/arguments_test.go @@ -24,6 +24,7 @@ func TestGetArguments(t *testing.T) { {"--follow-sitemap-xml", "https://foo.com"}, {"-t", "10", "https://foo.com"}, {"--timeout", "10", "https://foo.com"}, + {"--proxy", "localhost:8080", "https://foo.com"}, {"--skip-tls-verification", "https://foo.com"}, {"-v", "https://foo.com"}, {"--verbose", "https://foo.com"}, diff --git a/command.go b/command.go index 0f4936d2..d841c19c 100644 --- a/command.go +++ b/command.go @@ -51,6 +51,7 @@ func (c *command) runWithError(ss []string) (bool, error) { MaxConnectionsPerHost: args.MaxConnectionsPerHost, BufferSize: args.BufferSize, MaxRedirections: args.MaxRedirections, + Proxy: args.Proxy, SkipTLSVerification: args.SkipTLSVerification, Timeout: time.Duration(args.Timeout) * time.Second, }, diff --git a/configuration.go b/configuration.go index e05b98ec..90b6c745 100644 --- a/configuration.go +++ b/configuration.go @@ -3,7 +3,7 @@ package main import "time" const ( - version = "2.2.2" + version = "2.3.0" agentName = "muffet" concurrency = 1024 tcpTimeout = 5 * time.Second diff --git a/fasthttp_http_client_factory.go b/fasthttp_http_client_factory.go index f8bcfecf..67650deb 100644 --- a/fasthttp_http_client_factory.go +++ b/fasthttp_http_client_factory.go @@ -5,6 +5,7 @@ import ( "net" "github.com/valyala/fasthttp" + "github.com/valyala/fasthttp/fasthttpproxy" ) type fasthttpHTTPClientFactory struct { @@ -15,6 +16,14 @@ func newFasthttpHTTPClientFactory() *fasthttpHTTPClientFactory { } func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient { + d := func(addr string) (net.Conn, error) { + return fasthttp.DialTimeout(addr, tcpTimeout) + } + + if o.Proxy != "" { + d = fasthttpproxy.FasthttpHTTPDialerTimeout(o.Proxy, tcpTimeout) + } + return newFasthttpHTTPClient( &fasthttp.Client{ MaxConnsPerHost: o.MaxConnectionsPerHost, @@ -22,9 +31,7 @@ func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient { TLSConfig: &tls.Config{ InsecureSkipVerify: o.SkipTLSVerification, }, - Dial: func(addr string) (net.Conn, error) { - return fasthttp.DialTimeout(addr, tcpTimeout) - }, + Dial: d, }, o.MaxRedirections, o.Timeout, diff --git a/go.sum b/go.sum index d0309465..a1d8f16d 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= -github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= -github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= @@ -41,8 +39,7 @@ github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++Ja github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.16.0 h1:9zAqOYLl8Tuy3E5R6ckzGDJ1g8+pw15oQp2iL9Jl6gQ= -github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= +github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAkqg= github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945 h1:6Ju8pZBYFTN9FaV/JvNBiIHcsgEmP4z4laciqjfjY8E= @@ -50,23 +47,16 @@ github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945/go.mod h1:4vRFPPNYllgC golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= -golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg= golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8= -golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http_client_factory.go b/http_client_factory.go index 9301469a..0c655425 100644 --- a/http_client_factory.go +++ b/http_client_factory.go @@ -6,6 +6,7 @@ type httpClientOptions struct { MaxConnectionsPerHost, BufferSize, MaxRedirections int + Proxy string SkipTLSVerification bool Timeout time.Duration }