Merge pull request #74 from projectdiscovery/bugfix-rawhttp-uripath

fixing uripath

cmd/httpx/httpx.go

@@ -37,6 +37,7 @@ func main() {
 	httpxOptions.FollowHostRedirects = options.FollowHostRedirects
 	httpxOptions.HttpProxy = options.HttpProxy
 	httpxOptions.Unsafe = options.Unsafe
+	httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
 
 	var key, value string
 	httpxOptions.CustomHeaders = make(map[string]string)
@@ -302,9 +303,13 @@ type scanOptions struct {
 func analyze(hp *httpx.HTTPX, protocol string, domain string, port int, scanopts *scanOptions) Result {
 	retried := false
 retry:
-	URL := fmt.Sprintf("%s://%s%s", protocol, domain, scanopts.RequestURI)
+	URL := fmt.Sprintf("%s://%s", protocol, domain)
 	if port > 0 {
-		URL = fmt.Sprintf("%s://%s:%d%s", protocol, domain, port, scanopts.RequestURI)
+		URL = fmt.Sprintf("%s://%s:%d", protocol, domain, port)
+	}
+
+	if !scanopts.Unsafe {
+		URL += scanopts.RequestURI
 	}
 
 	req, err := hp.NewRequest(scanopts.Method, URL)

common/httpx/httpx.go

@@ -18,11 +18,12 @@ import (
 
 // HTTPX represent an instance of the library client
 type HTTPX struct {
-	client        *retryablehttp.Client
-	Filters       []Filter
-	Options       *Options
-	htmlPolicy    *bluemonday.Policy
-	CustomHeaders map[string]string
+	client          *retryablehttp.Client
+	Filters         []Filter
+	Options         *Options
+	htmlPolicy      *bluemonday.Policy
+	CustomHeaders   map[string]string
+	RequestOverride *RequestOverride
 }
 
 // New httpx instance
@@ -87,6 +88,7 @@ func New(options *Options) (*HTTPX, error) {
 
 	httpx.htmlPolicy = bluemonday.NewPolicy()
 	httpx.CustomHeaders = httpx.Options.CustomHeaders
+	httpx.RequestOverride = &options.RequestOverride
 
 	return httpx, nil
 }
@@ -153,9 +155,17 @@ func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
 	return &resp, nil
 }
 
+type RequestOverride struct {
+	URIPath string
+}
+
 // Do http request
 func (h *HTTPX) doUnsafe(req *retryablehttp.Request) (*http.Response, error) {
-	return rawhttp.Dor(req)
+	method := req.Method
+	headers := req.Header
+	url := req.URL.String()
+	body := req.Body
+	return rawhttp.DoRaw(method, url, h.RequestOverride.URIPath, headers, body)
 }
 
 // Verify the http calls and apply-cascade all the filters, as soon as one matches it returns true

common/httpx/option.go

@@ -17,6 +17,7 @@ type Options struct {
 	FollowHostRedirects bool
 	DefaultUserAgent    string
 	Unsafe              bool
+	RequestOverride     RequestOverride
 
 	HttpProxy  string
 	SocksProxy string

go.mod

@@ -10,10 +10,12 @@ require (
 	github.com/miekg/dns v1.1.31
 	github.com/projectdiscovery/gologger v1.0.1
 	github.com/projectdiscovery/mapcidr v0.0.4
-	github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087
+	github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.2.1
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
 	golang.org/x/net v0.0.0-20200822124328-c89045814202
+	golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect
 	golang.org/x/text v0.3.3
 )

go.sum

@@ -26,6 +26,8 @@ github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyW
 github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
 github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087 h1:FV+/XrXTWOaiW2hZVt/VWGxGBmOx/P9ChZnIkz7O96o=
 github.com/projectdiscovery/rawhttp v0.0.0-20200823205626-d8c41f52a087/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
+github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84 h1:2aO1hZXYAh/UIboBqXyBJ17bHnEWa3y/5rCrIUYqfD0=
+github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
 github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
@@ -39,6 +41,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -52,6 +56,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4=
+golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=