From 96a511ad4806d545d44f1fd5eb7d5919dd071f70 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 28 Dec 2021 07:46:42 +0100 Subject: [PATCH 1/2] Adding support for custom resolvers --- common/httpx/httpx.go | 3 +++ common/httpx/option.go | 1 + runner/options.go | 23 +++++++++++++++++++++++ runner/runner.go | 1 + 4 files changed, 28 insertions(+) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index b7c84d26..a0cc15dd 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -44,6 +44,9 @@ func New(options *Options) (*HTTPX, error) { fastdialerOpts.Deny = options.Deny fastdialerOpts.Allow = options.Allow fastdialerOpts.WithDialerHistory = true + if len(options.Resolvers) > 0 { + fastdialerOpts.BaseResolvers = options.Resolvers + } dialer, err := fastdialer.NewDialer(fastdialerOpts) if err != nil { return nil, fmt.Errorf("could not create resolver cache: %s", err) diff --git a/common/httpx/option.go b/common/httpx/option.go index 46d6070f..36d834a5 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -36,6 +36,7 @@ type Options struct { MaxResponseBodySizeToSave int64 MaxResponseBodySizeToRead int64 UnsafeURI string + Resolvers []string } // DefaultOptions contains the default options diff --git a/runner/options.go b/runner/options.go index 210f97bc..7631aa55 100644 --- a/runner/options.go +++ b/runner/options.go @@ -4,6 +4,7 @@ import ( "math" "os" "regexp" + "strings" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" @@ -197,6 +198,7 @@ type Options struct { Stream bool SkipDedupe bool ProbeAllIPS bool + Resolvers goflags.NormalizedStringSlice } // ParseOptions parses the command line options for application @@ -288,6 +290,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.Stream, "stream", "s", false, "Stream mode - start elaborating input targets without sorting"), flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "Disable dedupe input items (only used with stream mode)"), flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "Probe all the ips associated with same host"), + flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "List of resolvers (file or comma separated) - each resolver can be in the form protocol:ip|hostname:port - invalid files/resolvers ignored"), ) createGroup(flagSet, "debug", "Debug", @@ -370,6 +373,26 @@ func (options *Options) validateOptions() { gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err) } } + + var resolvers []string + for _, resolver := range options.Resolvers { + if fileutil.FileExists(resolver) { + chFile, err := fileutil.ReadFile(resolver) + if err != nil { + gologger.Fatal().Msgf("Couldn't process resolver file \"%s\": %s\n", resolver, err) + } + for line := range chFile { + resolvers = append(resolvers, line) + } + } else { + resolvers = append(resolvers, resolver) + } + } + options.Resolvers = resolvers + if len(options.Resolvers) > 0 { + gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ",")) + + } } // configureOutput configures the output on the screen diff --git a/runner/runner.go b/runner/runner.go index 9dc41313..98112e36 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -108,6 +108,7 @@ func New(options *Options) (*Runner, error) { if httpxOptions.MaxResponseBodySizeToSave > httpxOptions.MaxResponseBodySizeToRead { httpxOptions.MaxResponseBodySizeToSave = httpxOptions.MaxResponseBodySizeToRead } + httpxOptions.Resolvers = options.Resolvers var key, value string httpxOptions.CustomHeaders = make(map[string]string) From 481716ea007302a86384184effdb50322fee4547 Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 29 Dec 2021 15:31:47 +0530 Subject: [PATCH 2/2] readme update --- README.md | 4 +++- runner/options.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 668ae42d..53f503d2 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ OUTPUT: -csv Output in CSV format CONFIGURATIONS: + -r, -resolvers string[] List of custom resolvers (file or comma separated) -allow string[] Allowed list of IP/CIDR's to process (file or comma separated) -deny string[] Denied list of IP/CIDR's to process (file or comma separated) -random-agent Enable Random User-Agent to use (default true) @@ -323,7 +324,8 @@ https://support.hackerone.com - `vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes. - Unique flags should be used for specific use cases instead of running them as default with other flags. - When using `json` flag, all the information (default probes) included in the JSON output. - +- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (eg **udp:127.0.0.1:53**) +- Invalid custom resolvers/files are ignored. # Acknowledgement diff --git a/runner/options.go b/runner/options.go index 7631aa55..550f240b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -274,6 +274,7 @@ func ParseOptions() *Options { ) createGroup(flagSet, "configs", "Configurations", + flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "List of custom resolvers (file or comma separated)"), flagSet.Var(&options.Allow, "allow", "Allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "Denied list of IP/CIDR's to process (file or comma separated)"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), @@ -290,7 +291,6 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.Stream, "stream", "s", false, "Stream mode - start elaborating input targets without sorting"), flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "Disable dedupe input items (only used with stream mode)"), flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "Probe all the ips associated with same host"), - flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "List of resolvers (file or comma separated) - each resolver can be in the form protocol:ip|hostname:port - invalid files/resolvers ignored"), ) createGroup(flagSet, "debug", "Debug",