Merge pull request #179 from projectdiscovery/feature-multi-path

[Feature] Multi Paths + Random Agent

.golangci.yml

@@ -74,7 +74,6 @@ linters:
     - govet
     - ineffassign
     - interfacer
-    - maligned
     - misspell
     - nakedret
     - noctx

common/fileutil/fileutil.go

@@ -1,6 +1,9 @@
 package fileutil
 
-import "os"
+import (
+	"bufio"
+	"os"
+)
 
 // FileExists checks if a file exists and is not a directory
 func FileExists(filename string) bool {
@@ -31,3 +34,18 @@ func HasStdin() bool {
 
 	return isPipedFromChrDev || isPipedFromFIFO
 }
+
+// LoadFile content to slice
+func LoadFile(filename string) (lines []string) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return
+	}
+	defer f.Close() //nolint
+	s := bufio.NewScanner(f)
+	for s.Scan() {
+		lines = append(lines, s.Text())
+	}
+
+	return
+}

common/httpx/httpx.go

@@ -10,6 +10,7 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/corpix/uarand"
 	"github.com/microcosm-cc/bluemonday"
 	"github.com/projectdiscovery/cdncheck"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -254,4 +255,7 @@ func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]st
 			r.Host = value
 		}
 	}
+	if h.Options.RandomAgent {
+		r.Header.Set("User-Agent", uarand.GetRandom()) //nolint
+	}
 }

common/httpx/option.go

@@ -6,6 +6,7 @@ import (
 
 // Options contains configuration options for the client
 type Options struct {
+	RandomAgent      bool
 	DefaultUserAgent string
 	RequestOverride  RequestOverride
 	HTTPProxy        string

go.mod

@@ -3,6 +3,7 @@ module github.com/projectdiscovery/httpx
 go 1.14
 
 require (
+	github.com/corpix/uarand v0.1.1
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/microcosm-cc/bluemonday v1.0.4

go.sum

@@ -1,7 +1,13 @@
+github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic=
+github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/chris-ramon/douceur v0.2.0 h1:IDMEdxlEUUBYBKE4z/mJnFyVXox+MjuEVDJNN27glkU=
 github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpRXQKjTR8nIBE=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U=
+github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -26,6 +32,8 @@ github.com/microcosm-cc/bluemonday v1.0.4 h1:p0L+CTpo/PLFdkoPcJemLXG+fpMD7pYOoDE
 github.com/microcosm-cc/bluemonday v1.0.4/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w=
 github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

internal/runner/options.go

@@ -47,8 +47,39 @@ type scanOptions struct {
 	NoFallback             bool
 }
 
+func (s *scanOptions) Clone() *scanOptions {
+	return &scanOptions{
+		Methods:                s.Methods,
+		StoreResponseDirectory: s.StoreResponseDirectory,
+		RequestURI:             s.RequestURI,
+		RequestBody:            s.RequestBody,
+		VHost:                  s.VHost,
+		OutputTitle:            s.OutputTitle,
+		OutputStatusCode:       s.OutputStatusCode,
+		OutputLocation:         s.OutputLocation,
+		OutputContentLength:    s.OutputContentLength,
+		StoreResponse:          s.StoreResponse,
+		OutputServerHeader:     s.OutputServerHeader,
+		OutputWebSocket:        s.OutputWebSocket,
+		OutputWithNoColor:      s.OutputWithNoColor,
+		OutputMethod:           s.OutputMethod,
+		ResponseInStdout:       s.ResponseInStdout,
+		TLSProbe:               s.TLSProbe,
+		CSPProbe:               s.CSPProbe,
+		OutputContentType:      s.OutputContentType,
+		Unsafe:                 s.Unsafe,
+		Pipeline:               s.Pipeline,
+		HTTP2Probe:             s.HTTP2Probe,
+		OutputIP:               s.OutputIP,
+		OutputCName:            s.OutputCName,
+		OutputCDN:              s.OutputCDN,
+		OutputResponseTime:     s.OutputResponseTime,
+		PreferHTTPS:            s.PreferHTTPS,
+		NoFallback:             s.NoFallback,
+	}
+}
+
 // Options contains configuration options for chaos client.
-// nolint:maligned // ignore
 type Options struct {
 	CustomHeaders             customheader.CustomHeaders
 	CustomPorts               customport.CustomPorts
@@ -63,6 +94,8 @@ type Options struct {
 	InputFile                 string
 	Methods                   string
 	RequestURI                string
+	RequestURIs               string
+	requestURIs               []string
 	OutputMatchStatusCode     string
 	OutputMatchContentLength  string
 	OutputFilterStatusCode    string
@@ -111,6 +144,7 @@ type Options struct {
 	NoFallback                bool
 	protocol                  string
 	ShowStatistics            bool
+	RandomAgent               bool
 }
 
 // ParseOptions parses the command line options for application
@@ -147,6 +181,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains")
 	flag.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains")
 	flag.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')")
+	flag.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')")
 	flag.BoolVar(&options.OutputContentType, "content-type", false, "Extracts content-type")
 	flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match status code")
 	flag.StringVar(&options.OutputMatchStatusCode, "ml", "", "Match content length")
@@ -168,6 +203,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Output the response time")
 	flag.BoolVar(&options.NoFallback, "no-fallback", false, "If HTTPS on port 443 is successful on default configuration, probes also port 80 for HTTP")
 	flag.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)")
+	flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value")
 
 	flag.Parse()
 

internal/runner/runner.go

@@ -39,7 +39,7 @@ const (
 type Runner struct {
 	options  *Options
 	hp       *httpx.HTTPX
-	scanopts *scanOptions
+	scanopts scanOptions
 	hm       *hybrid.HybridMap
 	stats    clistats.StatisticsClient
 }
@@ -59,6 +59,7 @@ func New(options *Options) (*Runner, error) {
 	httpxOptions.Unsafe = options.Unsafe
 	httpxOptions.RequestOverride = httpx.RequestOverride{URIPath: options.RequestURI}
 	httpxOptions.CdnCheck = options.OutputCDN
+	httpxOptions.RandomAgent = options.RandomAgent
 
 	var key, value string
 	httpxOptions.CustomHeaders = make(map[string]string)
@@ -159,7 +160,7 @@ func New(options *Options) (*Runner, error) {
 		scanopts.OutputMethod = true
 	}
 
-	runner.scanopts = &scanopts
+	runner.scanopts = scanopts
 
 	if options.ShowStatistics {
 		runner.stats, err = clistats.New()
@@ -196,16 +197,28 @@ func (runner *Runner) prepareInput() {
 		gologger.Fatalf("No input provided")
 	}
 
+	// Check if the user requested multiple paths
+	if fileutil.FileExists(runner.options.RequestURIs) {
+		runner.options.requestURIs = fileutil.LoadFile(runner.options.RequestURIs)
+	} else if runner.options.RequestURIs != "" {
+		runner.options.requestURIs = strings.Split(runner.options.RequestURIs, ",")
+	}
+
 	numTargets := 0
 	for scanner.Scan() {
 		target := strings.TrimSpace(scanner.Text())
 		// Used just to get the exact number of targets
 		if _, ok := runner.hm.Get(target); ok {
 			continue
 		}
-		numTargets++
-		// nolint:errcheck // ignore
-		runner.hm.Set(target, nil)
+
+		if len(runner.options.requestURIs) > 0 {
+			numTargets += len(runner.options.requestURIs)
+		} else {
+			numTargets++
+		}
+
+		runner.hm.Set(target, nil) //nolint
 	}
 
 	if runner.options.InputFile != "" {
@@ -239,7 +252,7 @@ func makePrintCallback() func(stats clistats.StatisticsClient) {
 		builder.WriteRune('[')
 		startedAt, _ := stats.GetStatic("startedAt")
 		duration := time.Since(startedAt.(time.Time))
-		builder.WriteString(fmtDuration(duration))
+		builder.WriteString(clistats.FmtDuration(duration))
 		builder.WriteRune(']')
 
 		hosts, _ := stats.GetStatic("hosts")
@@ -301,8 +314,7 @@ func (runner *Runner) RunEnumeration() {
 			if err != nil {
 				gologger.Fatalf("Could not create output file '%s': %s\n", runner.options.Output, err)
 			}
-			//nolint:errcheck // this method needs a small refactor to reduce complexity
-			defer f.Close()
+			defer f.Close() //nolint
 		}
 		for r := range output {
 			if r.err != nil {
@@ -352,10 +364,22 @@ func (runner *Runner) RunEnumeration() {
 	wg := sizedwaitgroup.New(runner.options.Threads)
 
 	runner.hm.Scan(func(k, _ []byte) error {
+		var reqs int
+		if len(runner.options.requestURIs) > 0 {
+			for _, p := range runner.options.requestURIs {
+				scanopts := runner.scanopts.Clone()
+				scanopts.RequestURI = p
+				process(string(k), &wg, runner.hp, runner.options.protocol, scanopts, output)
+				reqs++
+			}
+		} else {
+			process(string(k), &wg, runner.hp, runner.options.protocol, &runner.scanopts, output)
+			reqs++
+		}
+
 		if runner.options.ShowStatistics {
-			runner.stats.IncrementCounter("requests", 1)
+			runner.stats.IncrementCounter("requests", reqs)
 		}
-		process(string(k), &wg, runner.hp, runner.options.protocol, runner.scanopts, output)
 		return nil
 	})