Added the Functionality of storing raw request response headers (#1671)

* implemented storing header

* Revert "chore(deps): bump github.com/quic-go/quic-go from 0.37.7 to 0.42.0"

This reverts commit a88d85a.

* implemented req/resp headers storing feature

* implemented req/resp headers storing feature

* update readme

* commit to last commit

* add `-ob, -omit-body` flag

---------

Co-authored-by: Sandeep Singh <[email protected]>
Co-authored-by: Doğan Can Bakır <[email protected]>
Co-authored-by: Mzack9999 <[email protected]>

README.md

@@ -134,7 +134,7 @@ MATCHERS:
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
    -ms, -match-string string[]        match response with specified string (-ms admin)
    -mr, -match-regex string[]         match response with specified regex (-mr admin)
-   -mcdn, -match-cdn string[]         match host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
+   -mcdn, -match-cdn string[]         match host with specified cdn provider (leaseweb, stackpath, cloudfront, fastly, google)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
    -mdc, -match-condition string      match response with dsl expression condition
 
@@ -151,7 +151,7 @@ FILTERS:
    -ffc, -filter-favicon string[]      filter response with specified favicon hash (-ffc 1494302000)
    -fs, -filter-string string[]        filter response with specified string (-fs admin)
    -fe, -filter-regex string[]         filter response with specified regex (-fe admin)
-   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
+   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (leaseweb, stackpath, cloudfront, fastly, google)
    -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
    -fdc, -filter-condition string      filter response with dsl expression condition
    -strip                              strips all tags in response. supported formats: html,xml (default html)
@@ -182,6 +182,7 @@ OUTPUT:
    -oa, -output-all                    filename to write output results in all formats
    -sr, -store-response                store http response to output directory
    -srd, -store-response-dir string    store http response to custom directory
+   -ob, -omit-body                     omit response body in output
    -csv                                store output in csv format
    -csvo, -csv-output-encoding string  define output encoding
    -j, -json                           store output in JSONL(ines) format
@@ -236,7 +237,7 @@ DEBUG:
 
 OPTIMIZATIONS:
    -nf, -no-fallback                  display both probed protocol (HTTPS and HTTP)
-   -nfs, -no-fallback-scheme          probe with protocol scheme specified in input 
+   -nfs, -no-fallback-scheme          probe with protocol scheme specified in input
    -maxhr, -max-host-error int        max error count per host before skipping remaining path/s (default 30)
    -e, -exclude string[]              exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex)
    -retries int                       number of retries

runner/options.go

@@ -55,6 +55,7 @@ type ScanOptions struct {
 	OutputLocation            bool
 	OutputContentLength       bool
 	StoreResponse             bool
+	OmitBody                  bool
 	OutputServerHeader        bool
 	OutputWebSocket           bool
 	OutputWithNoColor         bool
@@ -112,6 +113,7 @@ func (s *ScanOptions) Clone() *ScanOptions {
 		OutputLocation:            s.OutputLocation,
 		OutputContentLength:       s.OutputContentLength,
 		StoreResponse:             s.StoreResponse,
+		OmitBody:                  s.OmitBody,
 		OutputServerHeader:        s.OutputServerHeader,
 		OutputWebSocket:           s.OutputWebSocket,
 		OutputWithNoColor:         s.OutputWithNoColor,
@@ -164,6 +166,7 @@ type Options struct {
 	Output                    string
 	OutputAll                 bool
 	StoreResponseDir          string
+	OmitBody                  bool
 	HTTPProxy                 string
 	SocksProxy                string
 	InputFile                 string
@@ -415,6 +418,7 @@ func ParseOptions() *Options {
 		flagSet.BoolVarP(&options.OutputAll, "output-all", "oa", false, "filename to write output results in all formats"),
 		flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"),
 		flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"),
+		flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body in output"),
 		flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"),
 		flagSet.StringVarP(&options.CSVOutputEncoding, "csv-output-encoding", "csvo", "", "define output encoding"),
 		flagSet.BoolVarP(&options.JSONOutput, "json", "j", false, "store output in JSONL(ines) format"),

runner/runner.go

@@ -113,6 +113,7 @@ func New(options *Options) (*Runner, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not create wappalyzer client")
 	}
+
 	if options.StoreResponseDir != "" {
 		os.RemoveAll(filepath.Join(options.StoreResponseDir, "response", "index.txt"))
 		os.RemoveAll(filepath.Join(options.StoreResponseDir, "screenshot", "index_screenshot.txt"))
@@ -784,6 +785,7 @@ func (r *Runner) RunEnumeration() {
 			}
 			defer indexFile.Close() //nolint
 		}
+
 		if r.options.Screenshot {
 			var err error
 			indexScreenshotPath := filepath.Join(r.options.StoreResponseDir, "screenshot", "index_screenshot.txt")
@@ -814,6 +816,16 @@ func (r *Runner) RunEnumeration() {
 				continue
 			}
 
+			if indexFile != nil {
+				indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode))
+				_, _ = indexFile.WriteString(indexData)
+			}
+
+			if indexScreenshotFile != nil && resp.ScreenshotPathRel != "" {
+				indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.ScreenshotPathRel, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode))
+				_, _ = indexScreenshotFile.WriteString(indexData)
+			}
+
 			// apply matchers and filters
 			if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" {
 				if r.options.OutputMatchCondition != "" {
@@ -922,6 +934,10 @@ func (r *Runner) RunEnumeration() {
 			var responsePath, screenshotPath, screenshotPathRel string
 			// store response
 			if r.scanopts.StoreResponse || r.scanopts.StoreChain {
+				if r.scanopts.OmitBody {
+					resp.Raw = strings.Replace(resp.Raw, resp.ResponseBody, "", -1)
+				}
+
 				responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
 				// URL.EscapedString returns that can be used as filename
 				respRaw := resp.Raw
@@ -2008,6 +2024,42 @@ retry:
 		builder.WriteRune(']')
 	}
 
+	// store responses or chain in directory
+	domainFile := method + ":" + URL.EscapedString()
+	hash := hashes.Sha1([]byte(domainFile))
+	domainResponseFile := fmt.Sprintf("%s.txt", hash)
+	hostFilename := strings.ReplaceAll(URL.Host, ":", "_")
+
+	domainResponseBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "response")
+	responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename)
+
+	var responsePath string
+	// store response
+	if scanopts.StoreResponse || scanopts.StoreChain {
+		if r.options.OmitBody {
+			resp.Raw = strings.Replace(resp.Raw, string(resp.Data), "", -1)
+		}
+		responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile))
+		// URL.EscapedString returns that can be used as filename
+		respRaw := resp.Raw
+		reqRaw := requestDump
+		if len(respRaw) > scanopts.MaxResponseBodySizeToSave {
+			respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave]
+		}
+		data := reqRaw
+		if scanopts.StoreChain && resp.HasChain() {
+			data = append(data, append([]byte("\n"), []byte(resp.GetChain())...)...)
+		}
+		data = append(data, respRaw...)
+		data = append(data, []byte("\n\n\n")...)
+		data = append(data, []byte(fullURL)...)
+		_ = fileutil.CreateFolder(responseBaseDir)
+		writeErr := os.WriteFile(responsePath, data, 0644)
+		if writeErr != nil {
+			gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr)
+		}
+	}
+
 	parsed, err := r.parseURL(fullURL)
 	if err != nil {
 		return Result{URL: fullURL, Input: origInput, Err: errors.Wrap(err, "could not parse url")}