From edb9e1baaac9c355f9abd8fb963d08225b54ccfb Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 14 Sep 2020 19:40:19 +0200 Subject: [PATCH 1/3] adding string/regex match/filter --- cmd/httpx/httpx.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 5764d4ad..91ddd508 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "path" + "regexp" "strconv" "strings" "time" @@ -162,12 +163,24 @@ func main() { if len(options.filterContentLength) > 0 && slice.IntSliceContains(options.filterContentLength, r.ContentLength) { continue } + if options.filterRegex != nil && options.filterRegex.MatchString(r.Response) { + continue + } + if options.OutputFilterString != "" && strings.Contains(r.Response, options.OutputFilterString) { + continue + } if len(options.matchStatusCode) > 0 && !slice.IntSliceContains(options.matchStatusCode, r.StatusCode) { continue } if len(options.matchContentLength) > 0 && !slice.IntSliceContains(options.matchContentLength, r.ContentLength) { continue } + if options.matchRegex != nil && !options.matchRegex.MatchString(r.Response) { + continue + } + if options.OutputMatchString != "" && !strings.Contains(r.Response, options.OutputMatchString) { + continue + } row := r.str if options.JSONOutput { @@ -608,6 +621,12 @@ type Options struct { Debug bool Pipeline bool HTTP2Probe bool + OutputFilterString string + OutputMatchString string + OutputFilterRegex string + filterRegex *regexp.Regexp + OutputMatchRegex string + matchRegex *regexp.Regexp } // ParseOptions parses the command line options for application @@ -656,6 +675,10 @@ func ParseOptions() *Options { flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline") flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe") flag.BoolVar(&options.OutputIP, "ip", false, "Output target ip") + flag.StringVar(&options.OutputFilterString, "filter-string", "", "Filter String") + flag.StringVar(&options.OutputMatchString, "match-string", "", "Match string") + flag.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter Regex") + flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match Regex") flag.Parse() // Read the inputs and configure the logging @@ -695,6 +718,16 @@ func (options *Options) validateOptions() { if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil { gologger.Fatalf("Invalid value for filter content length option: %s\n", err) } + if options.OutputFilterRegex != "" { + if options.filterRegex, err = regexp.Compile(options.OutputFilterRegex); err != nil { + gologger.Fatalf("Invalid value for regex filter option: %s\n", err) + } + } + if options.OutputMatchRegex != "" { + if options.matchRegex, err = regexp.Compile(options.OutputMatchRegex); err != nil { + gologger.Fatalf("Invalid value for match regex option: %s\n", err) + } + } } // configureOutput configures the output on the screen From 84688d586869aae5aa0051959956b40a45d93c6f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 15 Sep 2020 06:57:52 +0200 Subject: [PATCH 2/3] fixed small bug --- cmd/httpx/httpx.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 91ddd508..5e5bd73e 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -163,10 +163,10 @@ func main() { if len(options.filterContentLength) > 0 && slice.IntSliceContains(options.filterContentLength, r.ContentLength) { continue } - if options.filterRegex != nil && options.filterRegex.MatchString(r.Response) { + if options.filterRegex != nil && options.filterRegex.MatchString(r.raw) { continue } - if options.OutputFilterString != "" && strings.Contains(r.Response, options.OutputFilterString) { + if options.OutputFilterString != "" && strings.Contains(r.raw, options.OutputFilterString) { continue } if len(options.matchStatusCode) > 0 && !slice.IntSliceContains(options.matchStatusCode, r.StatusCode) { @@ -175,10 +175,10 @@ func main() { if len(options.matchContentLength) > 0 && !slice.IntSliceContains(options.matchContentLength, r.ContentLength) { continue } - if options.matchRegex != nil && !options.matchRegex.MatchString(r.Response) { + if options.matchRegex != nil && !options.matchRegex.MatchString(r.raw) { continue } - if options.OutputMatchString != "" && !strings.Contains(r.Response, options.OutputMatchString) { + if options.OutputMatchString != "" && !strings.Contains(r.raw, options.OutputMatchString) { continue } @@ -518,6 +518,7 @@ retry: } return Result{ + raw: resp.Raw, URL: fullURL, ContentLength: resp.ContentLength, StatusCode: resp.StatusCode, @@ -541,6 +542,7 @@ retry: // Result of a scan type Result struct { + raw string URL string `json:"url"` ContentLength int `json:"content-length"` StatusCode int `json:"status-code"` From dc008dd9560c97148a363d0b155d32cd9346508f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 16 Sep 2020 07:38:48 +0200 Subject: [PATCH 3/3] case insensitive string filter/match --- cmd/httpx/httpx.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index d24b5d33..b0006f89 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -168,7 +168,7 @@ func main() { if options.filterRegex != nil && options.filterRegex.MatchString(r.raw) { continue } - if options.OutputFilterString != "" && strings.Contains(r.raw, options.OutputFilterString) { + if options.OutputFilterString != "" && strings.Contains(strings.ToLower(r.raw), options.OutputFilterString) { continue } if len(options.matchStatusCode) > 0 && !slice.IntSliceContains(options.matchStatusCode, r.StatusCode) { @@ -180,7 +180,7 @@ func main() { if options.matchRegex != nil && !options.matchRegex.MatchString(r.raw) { continue } - if options.OutputMatchString != "" && !strings.Contains(r.raw, options.OutputMatchString) { + if options.OutputMatchString != "" && !strings.Contains(strings.ToLower(r.raw), options.OutputMatchString) { continue } @@ -522,9 +522,9 @@ retry: if scanopts.OutputCName && len(cnames) > 0 { // Print only the first CNAME (full list in json) builder.WriteString(fmt.Sprintf(" [%s]", cnames[0])) - } - - isCDN := hp.CdnCheck(ip) + } + + isCDN := hp.CdnCheck(ip) if scanopts.OutputCDN && isCDN { builder.WriteString(" [cdn]") } @@ -716,7 +716,7 @@ func ParseOptions() *Options { flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match Regex") flag.BoolVar(&options.OutputCName, "cname", false, "Output first cname") flag.BoolVar(&options.OutputCDN, "cdn", false, "Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)") - + flag.Parse() // Read the inputs and configure the logging