Skip to content

Commit

Permalink
Merge pull request #1761 from projectdiscovery/support_multiple_values
Browse files Browse the repository at this point in the history 
Support multiple values
  • Loading branch information
@Mzack9999
Mzack9999 authored Jun 12, 2024
2 parents 8f9bb51 + 6811b18 commit e432123
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 71 deletions.
104 changes: 53 additions & 51 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,27 +95,28 @@ INPUT:
-u, -target string[] input target host(s) to probe

PROBES:
-sc, -status-code display response status-code
-cl, -content-length display response content-length
-ct, -content-type display response content-type
-location display response redirect location
-favicon display mmh3 hash for '/favicon.ico' file
-hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
-jarm display jarm fingerprint hash
-rt, -response-time display response time
-lc, -line-count display response body line count
-wc, -word-count display response body word count
-title display page title
-bp, -body-preview display first N characters of response body (default 100)
-server, -web-server display server name
-td, -tech-detect display technology in use based on wappalyzer dataset
-method display http request method
-websocket display server using websocket
-ip display host ip
-cname display host cname
-asn display host asn information
-cdn display cdn/waf in use (default true)
-probe display probe status
-sc, -status-code display response status-code
-cl, -content-length display response content-length
-ct, -content-type display response content-type
-location display response redirect location
-favicon display mmh3 hash for '/favicon.ico' file
-hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
-jarm display jarm fingerprint hash
-rt, -response-time display response time
-lc, -line-count display response body line count
-wc, -word-count display response body word count
-title display page title
-bp, -body-preview display first N characters of response body (default 100)
-server, -web-server display server name
-td, -tech-detect display technology in use based on wappalyzer dataset
-method display http request method
-websocket display server using websocket
-ip display host ip
-cname display host cname
-extract-fqdn, -efqdn get domain and subdomains from response body and header in jsonl/csv output
-asn display host asn information
-cdn display cdn/waf in use (default true)
-probe display probe status

HEADLESS:
-ss, -screenshot enable saving screenshot of the page using headless browser
Expand All @@ -131,15 +132,15 @@ MATCHERS:
-mlc, -match-line-count string match response body with specified line count (-mlc 423,532)
-mwc, -match-word-count string match response body with specified word count (-mwc 43,55)
-mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000)
-ms, -match-string string match response with specified string (-ms admin)
-mr, -match-regex string match response with specified regex (-mr admin)
-ms, -match-string string[] match response with specified string (-ms admin)
-mr, -match-regex string[] match response with specified regex (-mr admin)
-mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
-mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1')
-mdc, -match-condition string match response with dsl expression condition

EXTRACTOR:
-er, -extract-regex string[] display response content with matched regex
-ep, -extract-preset string[] display response content matched by a pre-defined regex (mail,url,ipv4)
-ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail)

FILTERS:
-fc, -filter-code string filter response with specified status code (-fc 403,401)
Expand All @@ -148,8 +149,8 @@ FILTERS:
-flc, -filter-line-count string filter response body with specified line count (-flc 423,532)
-fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532)
-ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000)
-fs, -filter-string string filter response with specified string (-fs admin)
-fe, -filter-regex string filter response with specified regex (-fe admin)
-fs, -filter-string string[] filter response with specified string (-fs admin)
-fe, -filter-regex string[] filter response with specified regex (-fe admin)
-fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google, leaseweb, stackpath)
-frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1')
-fdc, -filter-condition string filter response with dsl expression condition
Expand Down Expand Up @@ -193,31 +194,32 @@ OUTPUT:
-pr, -protocol string protocol to use (unknown, http11)

CONFIGURATIONS:
-config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
-auth configure projectdiscovery cloud (pdcp) api key (default true)
-r, -resolvers string[] list of custom resolver (file or comma separated)
-allow string[] allowed list of IP/CIDR's to process (file or comma separated)
-deny string[] denied list of IP/CIDR's to process (file or comma separated)
-sni, -sni-name string custom TLS SNI name
-random-agent enable Random User-Agent to use (default true)
-H, -header string[] custom http headers to send with request
-http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080)
-unsafe send raw requests skipping golang normalization
-resume resume scan using resume.cfg
-fr, -follow-redirects follow http redirects
-maxr, -max-redirects int max number of redirects to follow per host (default 10)
-fhr, -follow-host-redirects follow redirects on the same host
-rhsts, -respect-hsts respect HSTS response headers for redirect requests
-vhost-input get a list of vhosts as input
-x string request methods to probe, use 'all' to probe all HTTP methods
-body string post body to include in http request
-s, -stream stream mode - start elaborating input targets without sorting
-sd, -skip-dedupe disable dedupe input items (only used with stream mode)
-ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443
-ztls use ztls library with autofallback to standard one for tls13
-no-decode avoid decoding body
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
-no-stdin Disable Stdin processing
-config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)
-auth configure projectdiscovery cloud (pdcp) api key (default true)
-r, -resolvers string[] list of custom resolver (file or comma separated)
-allow string[] allowed list of IP/CIDR's to process (file or comma separated)
-deny string[] denied list of IP/CIDR's to process (file or comma separated)
-sni, -sni-name string custom TLS SNI name
-random-agent enable Random User-Agent to use (default true)
-H, -header string[] custom http headers to send with request
-http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080)
-unsafe send raw requests skipping golang normalization
-resume resume scan using resume.cfg
-fr, -follow-redirects follow http redirects
-maxr, -max-redirects int max number of redirects to follow per host (default 10)
-fhr, -follow-host-redirects follow redirects on the same host
-rhsts, -respect-hsts respect HSTS response headers for redirect requests
-vhost-input get a list of vhosts as input
-x string request methods to probe, use 'all' to probe all HTTP methods
-body string post body to include in http request
-s, -stream stream mode - start elaborating input targets without sorting
-sd, -skip-dedupe disable dedupe input items (only used with stream mode)
-ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https://host:443
-ztls use ztls library with autofallback to standard one for tls13
-no-decode avoid decoding body
-tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization
-no-stdin Disable Stdin processing
-hae, -http-api-endpoint string experimental http api endpoint

DEBUG:
-health-check, -hc run diagnostic check up
Expand Down
34 changes: 20 additions & 14 deletions runner/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,16 +180,16 @@ type Options struct {
InputRawRequest string
rawRequest string
RequestBody string
OutputFilterString string
OutputMatchString string
OutputFilterRegex string
OutputMatchRegex string
OutputFilterString goflags.StringSlice
OutputMatchString goflags.StringSlice
OutputFilterRegex goflags.StringSlice
OutputMatchRegex goflags.StringSlice
Retries int
Threads int
Timeout int
Delay time.Duration
filterRegex *regexp.Regexp
matchRegex *regexp.Regexp
filterRegexes []*regexp.Regexp
matchRegexes []*regexp.Regexp
VHost bool
VHostInput bool
Smuggling bool
Expand Down Expand Up @@ -359,8 +359,8 @@ func ParseOptions() *Options {
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"),
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"),
flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions),
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"),
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"),
flagSet.StringSliceVarP(&options.OutputMatchString, "match-string", "ms", nil, "match response with specified string (-ms admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputMatchRegex, "match-regex", "mr", nil, "match response with specified regex (-mr admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"),
flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"),
Expand All @@ -378,8 +378,8 @@ func ParseOptions() *Options {
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"),
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"),
flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-ffc 1494302000)", goflags.NormalizedStringSliceOptions),
flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"),
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"),
flagSet.StringSliceVarP(&options.OutputFilterString, "filter-string", "fs", nil, "filter response with specified string (-fs admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputFilterRegex, "filter-regex", "fe", nil, "filter response with specified regex (-fe admin)", goflags.NormalizedStringSliceOptions),
flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", cdncheck.DefaultCDNProviders), goflags.NormalizedStringSliceOptions),
flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"),
flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"),
Expand Down Expand Up @@ -610,16 +610,22 @@ func (options *Options) ValidateOptions() error {
if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil {
return errors.Wrap(err, "Invalid value for filter content length option")
}
if options.OutputFilterRegex != "" {
if options.filterRegex, err = regexp.Compile(options.OutputFilterRegex); err != nil {
for _, filterRegexStr := range options.OutputFilterRegex {
filterRegex, err := regexp.Compile(filterRegexStr)
if err != nil {
return errors.Wrap(err, "Invalid value for regex filter option")
}
options.filterRegexes = append(options.filterRegexes, filterRegex)
}
if options.OutputMatchRegex != "" {
if options.matchRegex, err = regexp.Compile(options.OutputMatchRegex); err != nil {

for _, matchRegexStr := range options.OutputMatchRegex {
matchRegex, err := regexp.Compile(matchRegexStr)
if err != nil {
return errors.Wrap(err, "Invalid value for match regex option")
}
options.matchRegexes = append(options.matchRegexes, matchRegex)
}

if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
return errors.Wrap(err, "Invalid value for match lines count option")
}
Expand Down
30 changes: 24 additions & 6 deletions runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -846,10 +846,19 @@ func (r *Runner) RunEnumeration() {
if len(r.options.filterWordsCount) > 0 && sliceutil.Contains(r.options.filterWordsCount, resp.Words) {
continue
}
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.Raw) {
continue
if r.options.filterRegexes != nil {
shouldContinue := false
for _, filterRegex := range r.options.filterRegexes {
if filterRegex.MatchString(resp.Raw) {
shouldContinue = true
break
}
}
if shouldContinue {
continue
}
}
if r.options.OutputFilterString != "" && stringsutil.ContainsAnyI(resp.Raw, r.options.OutputFilterString) {
if len(r.options.OutputFilterString) > 0 && stringsutil.EqualFoldAny(resp.Raw, r.options.OutputFilterString...) {
continue
}
if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) {
Expand All @@ -861,10 +870,19 @@ func (r *Runner) RunEnumeration() {
if len(r.options.matchContentLength) > 0 && !sliceutil.Contains(r.options.matchContentLength, resp.ContentLength) {
continue
}
if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.Raw) {
continue
if r.options.matchRegexes != nil {
shouldContinue := false
for _, matchRegex := range r.options.matchRegexes {
if !matchRegex.MatchString(resp.Raw) {
shouldContinue = true
break
}
}
if shouldContinue {
continue
}
}
if r.options.OutputMatchString != "" && !stringsutil.ContainsAnyI(resp.Raw, r.options.OutputMatchString) {
if len(r.options.OutputMatchString) > 0 && !stringsutil.ContainsAnyI(resp.Raw, r.options.OutputMatchString...) {
continue
}
if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) {
Expand Down

0 comments on commit e432123

Please sign in to comment.