Skip to content

Commit

Permalink
HTTP proxy support (#132)
Browse files Browse the repository at this point in the history 
* Parse proxy argument

* Use proxy

* Change test proxy url

* Add value name

* Bump version

* Update help message

* Update help message again
  • Loading branch information
@raviqqe
raviqqe authored Nov 11, 2020
1 parent a84b545 commit e135cc7
Show file tree
Hide file tree
Showing 8 changed files with 20 additions and 18 deletions.
1 change: 1 addition & 0 deletions .snapshots/TestHelp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ Application Options:
-t, --timeout=<seconds> Timeout for HTTP requests in
seconds (default: 10)
-v, --verbose Show successful results too
--proxy=<host> HTTP proxy host
--skip-tls-verification Skip TLS certificate verification
--one-page-only Only check links found in the given
URL
Expand Down
1 change: 1 addition & 0 deletions arguments.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ type arguments struct {
MaxRedirections int `short:"r" long:"max-redirections" value-name:"<count>" default:"64" description:"Maximum number of redirections"`
Timeout int `short:"t" long:"timeout" value-name:"<seconds>" default:"10" description:"Timeout for HTTP requests in seconds"`
Verbose bool `short:"v" long:"verbose" description:"Show successful results too"`
Proxy string `long:"proxy" value-name:"<host>" description:"HTTP proxy host"`
SkipTLSVerification bool `long:"skip-tls-verification" description:"Skip TLS certificate verification"`
OnePageOnly bool `long:"one-page-only" description:"Only check links found in the given URL"`
Color color `long:"color" description:"Color output" choice:"auto" choice:"always" choice:"never" default:"auto"`
Expand Down
1 change: 1 addition & 0 deletions arguments_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ func TestGetArguments(t *testing.T) {
{"--follow-sitemap-xml", "https://foo.com"},
{"-t", "10", "https://foo.com"},
{"--timeout", "10", "https://foo.com"},
{"--proxy", "localhost:8080", "https://foo.com"},
{"--skip-tls-verification", "https://foo.com"},
{"-v", "https://foo.com"},
{"--verbose", "https://foo.com"},
Expand Down
1 change: 1 addition & 0 deletions command.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func (c *command) runWithError(ss []string) (bool, error) {
MaxConnectionsPerHost: args.MaxConnectionsPerHost,
BufferSize: args.BufferSize,
MaxRedirections: args.MaxRedirections,
Proxy: args.Proxy,
SkipTLSVerification: args.SkipTLSVerification,
Timeout: time.Duration(args.Timeout) * time.Second,
},
Expand Down
2 changes: 1 addition & 1 deletion configuration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package main
import "time"

const (
version = "2.2.2"
version = "2.3.0"
agentName = "muffet"
concurrency = 1024
tcpTimeout = 5 * time.Second
Expand Down
13 changes: 10 additions & 3 deletions fasthttp_http_client_factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"net"

"github.com/valyala/fasthttp"
"github.com/valyala/fasthttp/fasthttpproxy"
)

type fasthttpHTTPClientFactory struct {
Expand All @@ -15,16 +16,22 @@ func newFasthttpHTTPClientFactory() *fasthttpHTTPClientFactory {
}

func (*fasthttpHTTPClientFactory) Create(o httpClientOptions) httpClient {
d := func(addr string) (net.Conn, error) {
return fasthttp.DialTimeout(addr, tcpTimeout)
}

if o.Proxy != "" {
d = fasthttpproxy.FasthttpHTTPDialerTimeout(o.Proxy, tcpTimeout)
}

return newFasthttpHTTPClient(
&fasthttp.Client{
MaxConnsPerHost: o.MaxConnectionsPerHost,
ReadBufferSize: o.BufferSize,
TLSConfig: &tls.Config{
InsecureSkipVerify: o.SkipTLSVerification,
},
Dial: func(addr string) (net.Conn, error) {
return fasthttp.DialTimeout(addr, tcpTimeout)
},
Dial: d,
},
o.MaxRedirections,
o.Timeout,
Expand Down
18 changes: 4 additions & 14 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4=
github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc=
github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw=
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
Expand All @@ -41,32 +39,24 @@ github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++Ja
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.16.0 h1:9zAqOYLl8Tuy3E5R6ckzGDJ1g8+pw15oQp2iL9Jl6gQ=
github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
github.com/valyala/fasthttp v1.17.0 h1:P8/koH4aSnJ4xbd0cUUFEGQs3jQqIxoDDyRQrUiAkqg=
github.com/valyala/fasthttp v1.17.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945 h1:6Ju8pZBYFTN9FaV/JvNBiIHcsgEmP4z4laciqjfjY8E=
github.com/yhat/scrape v0.0.0-20161128144610-24b7890b0945/go.mod h1:4vRFPPNYllgCacoj+0FoKOjTW68rUhEfqPLiEJaK2w8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0 h1:5kGOVHlq0euqwzgTC9Vu15p6fV1Wi0ArVi8da2urnVg=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8=
golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
1 change: 1 addition & 0 deletions http_client_factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ type httpClientOptions struct {
MaxConnectionsPerHost,
BufferSize,
MaxRedirections int
Proxy string
SkipTLSVerification bool
Timeout time.Duration
}
Expand Down

0 comments on commit e135cc7

Please sign in to comment.