Bufix request response log

.golangci.yml

@@ -73,7 +73,6 @@ linters:
     - govet
     - ineffassign
     - interfacer
-    - maligned
     - misspell
     - nakedret
     - noctx

cmd/mitmrelay/mitmrelay.go

@@ -38,7 +38,7 @@ type Options struct {
 func httpserver(addr string) error {
 	// echo server
 	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
-		io.Copy(w, req.Body)
+		io.Copy(w, req.Body) //nolint
 	})
 
 	return http.ListenAndServe(addr, nil)
@@ -117,8 +117,8 @@ func main() {
 		proxyOpts.Timeout = time.Duration(options.Timeout) * time.Second
 	}
 
-	go httpserver(options.HTTPListenerAddress)
-	go dnsserver(options.ListenDNSAddr, options.DNSFallbackResolver, options.DNSMapping)
+	go httpserver(options.HTTPListenerAddress)                                           //nolint
+	go dnsserver(options.ListenDNSAddr, options.DNSFallbackResolver, options.DNSMapping) //nolint
 
 	var wgproxies sync.WaitGroup
 

cmd/replay/replay.go

@@ -81,7 +81,7 @@ func main() {
 			w.Header().Add(k, strings.Join(v, "; "))
 		}
 		w.WriteHeader(response.StatusCode)
-		_, _ = io.Copy(w, response.Body)
+		io.Copy(w, response.Body) //nolint
 	})
 	go func() {
 		if err := http.ListenAndServe(":80", nil); err != nil {

go.mod

@@ -1,6 +1,6 @@
 module github.com/projectdiscovery/proxify
 
-go 1.14
+go 1.15
 
 require (
 	github.com/Knetic/govaluate v3.0.0+incompatible // indirect
@@ -14,5 +14,5 @@ require (
 	github.com/projectdiscovery/tinydns v0.0.1
 	github.com/rs/xid v1.2.1
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	golang.org/x/net v0.0.0-20201209123823-ac852fbbde11
+	golang.org/x/net v0.0.0-20201224014010-6772e930b67b
 )

go.sum

@@ -2,6 +2,7 @@ github.com/Knetic/govaluate v1.5.0 h1:L4MyqdJSld9xr2eZcZHCWLfeIX2SBjqrwIKG1pcm/+
 github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
 github.com/elazarl/goproxy v0.0.0-20201021153353-00ad82a08272 h1:Am81SElhR3XCQBunTisljzNkNese2T1FiV8jP79+dqg=
@@ -11,13 +12,20 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -30,6 +38,8 @@ github.com/projectdiscovery/fastdialer v0.0.2 h1:0VUoHhtUt/HThHUUwbWBxTnFI+tM13R
 github.com/projectdiscovery/fastdialer v0.0.2/go.mod h1:wjSQICydWE54N49Lcx9nnh5OmtsRwIcLgiVT3GT2zgA=
 github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
+github.com/projectdiscovery/gologger v1.1.2 h1:kjHYuJeTnjbfNS4gvP1K8YzP6OMkr/vgg5ogNG15OG0=
+github.com/projectdiscovery/gologger v1.1.2/go.mod h1:jdXflz3TLB8bcVNzb0v26TztI9KPz8Lr4BVdUhNUs6E=
 github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6KR1wog=
 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0=
 github.com/projectdiscovery/mapsutil v0.0.1 h1:hXIQUPvU+5HtPzzyaLvsPg4R/39N77CCAL0ezYhFdgw=
@@ -44,6 +54,7 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@@ -55,15 +66,14 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw=
+golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/runner/options.go

@@ -28,6 +28,8 @@ type Options struct {
 	ResponseMatchReplaceDSL string
 	UpstreamHTTPProxy       string
 	UpstreamSocks5Proxy     string
+	DumpRequest             bool
+	DumpResponse            bool
 }
 
 func ParseOptions() *Options {
@@ -54,7 +56,9 @@ func ParseOptions() *Options {
 	flag.StringVar(&options.ListenDNSAddr, "dns-addr", "", "Listen DNS Ip and port (ip:port)")
 	flag.StringVar(&options.DNSMapping, "dns-mapping", "", "DNS A mapping (eg domain:ip,domain:ip,..)")
 	flag.StringVar(&options.UpstreamHTTPProxy, "http-proxy", "", "Upstream HTTP Proxy (eg http://proxyip:proxyport")
-	flag.StringVar(&options.UpstreamSocks5Proxy, "socks5-proxy", "", "Upstream SOCKS5 Proxy (eg socks5://proxyip:proxyport")
+	flag.StringVar(&options.UpstreamSocks5Proxy, "socks5-proxy", "", "Upstream SOCKS5 Proxy (eg socks5://proxyip:proxyport)")
+	flag.BoolVar(&options.DumpRequest, "dump-req", false, "Dump requests in separate files")
+	flag.BoolVar(&options.DumpResponse, "dump-resp", false, "Dump responses in separate files")
 
 	flag.Parse()
 	_ = os.MkdirAll(options.Directory, os.ModePerm)

internal/runner/runner.go

@@ -31,6 +31,8 @@ func NewRunner(options *Options) (*Runner, error) {
 		DNSFallbackResolver:     options.DNSFallbackResolver,
 		RequestMatchReplaceDSL:  options.RequestMatchReplaceDSL,
 		ResponseMatchReplaceDSL: options.ResponseMatchReplaceDSL,
+		DumpRequest:             options.DumpRequest,
+		DumpResponse:            options.DumpResponse,
 	})
 	if err != nil {
 		return nil, err

logger.go

@@ -6,13 +6,21 @@ import (
 	"net/http/httputil"
 	"os"
 	"path"
+	"strings"
 
 	"github.com/projectdiscovery/gologger"
 )
 
+const (
+	dataWithNewLine    = "%s\n\n"
+	dataWithoutNewLine = "%s"
+)
+
 type OptionsLogger struct {
 	Verbose      bool
 	OutputFolder string
+	DumpRequest  bool
+	DumpResponse bool
 }
 
 type OutputData struct {
@@ -30,7 +38,7 @@ func NewLogger(options *OptionsLogger) *Logger {
 		options:    options,
 		asyncqueue: make(chan OutputData, 1000),
 	}
-	_ = logger.createOutputFolder()
+	logger.createOutputFolder() //nolint
 	go logger.AsyncWrite()
 	return logger
 }
@@ -43,21 +51,45 @@ func (l *Logger) createOutputFolder() error {
 }
 
 func (l *Logger) AsyncWrite() {
+	var (
+		format     string
+		partSuffix string
+		ext        string
+	)
 	for outputdata := range l.asyncqueue {
-		destFile := path.Join(l.options.OutputFolder, fmt.Sprintf("%s-%s", outputdata.userdata.host, outputdata.userdata.id))
+		if !l.options.DumpRequest && !l.options.DumpResponse {
+			partSuffix = ""
+			ext = ""
+		} else if l.options.DumpRequest && !outputdata.userdata.hasResponse {
+			partSuffix = ".request"
+			ext = ".txt"
+		} else if l.options.DumpResponse && outputdata.userdata.hasResponse {
+			partSuffix = ".response"
+			ext = ".txt"
+		} else {
+			continue
+		}
+		destFile := path.Join(l.options.OutputFolder, fmt.Sprintf("%s%s-%s%s", outputdata.userdata.host, partSuffix, outputdata.userdata.id, ext))
 		// if it's a response and file doesn't exist skip
 		f, err := os.OpenFile(destFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			continue
 		}
-		fmt.Fprintf(f, "%s", outputdata.data)
+
+		format = dataWithoutNewLine
+		if !strings.HasSuffix(string(outputdata.data), "\n") {
+			format = dataWithNewLine
+		}
+
+		fmt.Fprintf(f, format, outputdata.data)
+
 		f.Close()
-		if outputdata.userdata.hasResponse {
+		if outputdata.userdata.hasResponse && !(l.options.DumpRequest || l.options.DumpResponse) {
 			outputFileName := destFile + ".txt"
 			if outputdata.userdata.match {
 				outputFileName = destFile + ".match.txt"
 			}
-			_ = os.Rename(destFile, outputFileName)
+			os.Rename(destFile, outputFileName) //nolint
 		}
 	}
 }
@@ -72,7 +104,7 @@ func (l *Logger) LogRequest(req *http.Request, userdata UserData) error {
 	}
 
 	if l.options.Verbose {
-		gologger.Silentf(string(reqdump))
+		gologger.Silentf("%s", string(reqdump))
 	}
 
 	return nil
@@ -91,7 +123,7 @@ func (l *Logger) LogResponse(resp *http.Response, userdata UserData) error {
 	}
 
 	if l.options.Verbose {
-		gologger.Silentf(string(respdump))
+		gologger.Silentf("%s", string(respdump))
 	}
 	return nil
 }

pkg/certs/ca.go

@@ -84,7 +84,7 @@ func (m *Manager) createAuthority(certPath, keyPath string) error {
 var errExpiredCert = errors.New("expired cert error")
 
 // readCertificateDisk reads a certificate and key file from disk
-func (m *Manager) readCertificateDisk(certFile string, keyFile string) (*tls.Certificate, error) {
+func (m *Manager) readCertificateDisk(certFile, keyFile string) (*tls.Certificate, error) {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		return nil, err

proxy.go

@@ -33,6 +33,8 @@ type OnResponseFunc func(*http.Response, *goproxy.ProxyCtx) *http.Response
 type OnConnectFunc func(string, *goproxy.ProxyCtx) (*goproxy.ConnectAction, string)
 
 type Options struct {
+	DumpRequest             bool
+	DumpResponse            bool
 	Silent                  bool
 	Verbose                 bool
 	CertCacheSize           int
@@ -85,7 +87,7 @@ func (p *Proxy) OnRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Reque
 		req = p.MatchReplaceRequest(req)
 	}
 
-	_ = p.logger.LogRequest(req, userdata)
+	p.logger.LogRequest(req, userdata) //nolint
 	ctx.UserData = userdata
 
 	return req, nil
@@ -105,7 +107,7 @@ func (p *Proxy) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Res
 		p.MatchReplaceResponse(resp)
 	}
 
-	_ = p.logger.LogResponse(resp, userdata)
+	p.logger.LogResponse(resp, userdata) //nolint
 	ctx.UserData = userdata
 	return resp
 }
@@ -276,6 +278,8 @@ func NewProxy(options *Options) (*Proxy, error) {
 	logger := NewLogger(&OptionsLogger{
 		Verbose:      options.Verbose,
 		OutputFolder: options.OutputDirectory,
+		DumpRequest:  options.DumpRequest,
+		DumpResponse: options.DumpResponse,
 	})
 
 	var tdns *tinydns.TinyDNS

socket.go

@@ -186,8 +186,8 @@ func (p *SocketConn) fullduplex() {
 	go p.pipe(p.lconn, p.rconn)
 	go p.pipe(p.rconn, p.lconn)
 	if p.Timeout > 0 {
-		p.lconn.SetDeadline(time.Now().Add(p.Timeout))
-		p.rconn.SetDeadline(time.Now().Add(p.Timeout))
+		p.lconn.SetDeadline(time.Now().Add(p.Timeout)) //nolint
+		p.rconn.SetDeadline(time.Now().Add(p.Timeout)) //nolint
 	}
 
 	//wait for close...