Skip to content

Commit

Permalink
Merge pull request #1147 from projectdiscovery/sdk-code-improvements
Browse files Browse the repository at this point in the history 
SDK Improvements
  • Loading branch information
@Mzack9999
Mzack9999 authored May 6, 2023
2 parents 6baaf8e + b3b14b6 commit 614f997
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 38 deletions.
6 changes: 3 additions & 3 deletions runner/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const (
// OnResultCallback (hostResult)
type OnResultCallback func(Result)

type scanOptions struct {
type ScanOptions struct {
Methods []string
StoreResponseDirectory string
RequestURI string
Expand Down Expand Up @@ -87,8 +87,8 @@ type scanOptions struct {
UseInstalledChrome bool
}

func (s *scanOptions) Clone() *scanOptions {
return &scanOptions{
func (s *ScanOptions) Clone() *ScanOptions {
return &ScanOptions{
Methods: s.Methods,
StoreResponseDirectory: s.StoreResponseDirectory,
RequestURI: s.RequestURI,
Expand Down
52 changes: 26 additions & 26 deletions runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ type Runner struct {
hp *httpx.HTTPX
wappalyzer *wappalyzer.Wappalyze
fastdialer *fastdialer.Dialer
scanopts scanOptions
scanopts ScanOptions
hm *hybrid.HybridMap
stats clistats.StatisticsClient
ratelimiter ratelimit.Limiter
Expand Down Expand Up @@ -162,7 +162,7 @@ func New(options *Options) (*Runner, error) {
gologger.Fatal().Msgf("Could not create httpx instance: %s\n", err)
}

var scanopts scanOptions
var scanopts ScanOptions

if options.InputRawRequest != "" {
var rawRequest []byte
Expand Down Expand Up @@ -677,12 +677,12 @@ func (r *Runner) RunEnumeration() {
}

for resp := range output {
if resp.err != nil {
if resp.Err != nil {
// Change the error message if any port value passed explicitly
if url, err := r.parseURL(resp.URL); err == nil && url.Port() != "" {
resp.err = errors.New(strings.ReplaceAll(resp.err.Error(), "address", "port"))
resp.Err = errors.New(strings.ReplaceAll(resp.Err.Error(), "address", "port"))
}
gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.err)
gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.Err)
}
if resp.str == "" {
continue
Expand Down Expand Up @@ -746,10 +746,10 @@ func (r *Runner) RunEnumeration() {
if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) {
continue
}
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) {
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.Raw) {
continue
}
if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputFilterString)) {
if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.Raw), strings.ToLower(r.options.OutputFilterString)) {
continue
}
if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) {
Expand All @@ -761,10 +761,10 @@ func (r *Runner) RunEnumeration() {
if len(r.options.matchContentLength) > 0 && !slice.IntSliceContains(r.options.matchContentLength, resp.ContentLength) {
continue
}
if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.raw) {
if r.options.matchRegex != nil && !r.options.matchRegex.MatchString(resp.Raw) {
continue
}
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.Raw), strings.ToLower(r.options.OutputMatchString)) {
continue
}
if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) {
Expand Down Expand Up @@ -911,15 +911,15 @@ func (r *Runner) RunEnumeration() {
wgoutput.Wait()
}

func (r *Runner) GetScanOpts() scanOptions {
func (r *Runner) GetScanOpts() ScanOptions {
return r.scanopts
}

func (r *Runner) Process(t string, wg *sizedwaitgroup.SizedWaitGroup, protocol string, scanopts *scanOptions, output chan Result) {
func (r *Runner) Process(t string, wg *sizedwaitgroup.SizedWaitGroup, protocol string, scanopts *ScanOptions, output chan Result) {
r.process(t, wg, r.hp, protocol, scanopts, output)
}

func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) {
func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) {
protocols := []string{protocol}
if scanopts.NoFallback || protocol == httpx.HTTPandHTTPS {
protocols = []string{httpx.HTTPS, httpx.HTTP}
Expand Down Expand Up @@ -1062,7 +1062,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target {
return results
}

func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *scanOptions) Result {
func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *ScanOptions) Result {
origProtocol := protocol
if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS {
protocol = httpx.HTTPS
Expand All @@ -1074,22 +1074,22 @@ retry:
}
URL, err := r.parseURL(target.Host)
if err != nil {
return Result{URL: target.Host, Input: origInput, err: err}
return Result{URL: target.Host, Input: origInput, Err: err}
}

// check if we have to skip the host:port as a result of a previous failure
hostPort := net.JoinHostPort(URL.Host, URL.Port())
if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) {
numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort)
if err == nil && numberOfErrors >= r.options.HostMaxErrors {
return Result{URL: target.Host, err: errors.New("skipping as previously unresponsive")}
return Result{URL: target.Host, Err: errors.New("skipping as previously unresponsive")}
}
}

// check if the combination host:port should be skipped if belonging to a cdn
if r.skipCDNPort(URL.Host, URL.Port()) {
gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port())
return Result{URL: target.Host, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")}
return Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")}
}

URL.Scheme = protocol
Expand Down Expand Up @@ -1117,7 +1117,7 @@ retry:
req, err = hp.NewRequest(method, URL.String())
}
if err != nil {
return Result{URL: URL.String(), Input: origInput, err: err}
return Result{URL: URL.String(), Input: origInput, Err: err}
}

if target.CustomHost != "" {
Expand Down Expand Up @@ -1158,7 +1158,7 @@ retry:
var errDump error
requestDump, errDump = rawhttp.DumpRequestRaw(req.Method, req.URL.String(), reqURI, req.Header, req.Body, rawhttp.DefaultOptions)
if errDump != nil {
return Result{URL: URL.String(), Input: origInput, err: errDump}
return Result{URL: URL.String(), Input: origInput, Err: errDump}
}
} else {
// Create a copy on the fly of the request body
Expand All @@ -1169,7 +1169,7 @@ retry:
var errDump error
requestDump, errDump = httputil.DumpRequestOut(req.Request, true)
if errDump != nil {
return Result{URL: URL.String(), Input: origInput, err: errDump}
return Result{URL: URL.String(), Input: origInput, Err: errDump}
}
// The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty
// Otherwise redirects like 307/308 would fail (as they require the body to be sent along)
Expand All @@ -1181,7 +1181,7 @@ retry:
// fix the final output url
fullURL := req.URL.String()
if parsedURL, errParse := r.parseURL(fullURL); errParse != nil {
return Result{URL: URL.String(), Input: origInput, err: errParse}
return Result{URL: URL.String(), Input: origInput, Err: errParse}
} else {
if r.options.Unsafe {
parsedURL.Path = reqURI
Expand Down Expand Up @@ -1249,9 +1249,9 @@ retry:
}

if r.options.Probe {
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err, Failed: err != nil, Error: errString, str: builder.String()}
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, str: builder.String()}
} else {
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), err: err}
return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err}
}
}

Expand Down Expand Up @@ -1652,7 +1652,7 @@ retry:

parsed, err := r.parseURL(fullURL)
if err != nil {
return Result{URL: fullURL, Input: origInput, err: errors.Wrap(err, "could not parse url")}
return Result{URL: fullURL, Input: origInput, Err: errors.Wrap(err, "could not parse url")}
}

finalPort := parsed.Port()
Expand Down Expand Up @@ -1704,7 +1704,7 @@ retry:
Scheme: parsed.Scheme,
Port: finalPort,
Path: finalPath,
raw: resp.Raw,
Raw: resp.Raw,
URL: fullURL,
Input: origInput,
ContentLength: resp.ContentLength,
Expand Down Expand Up @@ -1829,7 +1829,7 @@ func (r *Runner) SaveResumeConfig() error {
}

// JSON the result
func (r Result) JSON(scanopts *scanOptions) string { //nolint
func (r Result) JSON(scanopts *ScanOptions) string { //nolint
if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave {
r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave]
}
Expand Down Expand Up @@ -1864,7 +1864,7 @@ func (r Result) CSVHeader() string { //nolint
}

// CSVRow the CSV Row
func (r Result) CSVRow(scanopts *scanOptions) string { //nolint
func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint
if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave {
r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave]
}
Expand Down
18 changes: 9 additions & 9 deletions runner/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,20 @@ func (o AsnResponse) String() string {

// Result of a scan
type Result struct {
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"`
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
err error
Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"`
ASN *AsnResponse `json:"asn,omitempty" csv:"asn"`
Err error `json:"-" csv:"-"`
CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"`
TLSData *clients.Response `json:"tls,omitempty" csv:"tls"`
Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"`
ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"`
CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"`
Port string `json:"port,omitempty" csv:"port"`
raw string
URL string `json:"url,omitempty" csv:"url"`
Input string `json:"input,omitempty" csv:"input"`
Location string `json:"location,omitempty" csv:"location"`
Title string `json:"title,omitempty" csv:"title"`
Raw string `json:"-" csv:"-"`
URL string `json:"url,omitempty" csv:"url"`
Input string `json:"input,omitempty" csv:"input"`
Location string `json:"location,omitempty" csv:"location"`
Title string `json:"title,omitempty" csv:"title"`
str string
Scheme string `json:"scheme,omitempty" csv:"scheme"`
Error string `json:"error,omitempty" csv:"error"`
Expand Down Expand Up @@ -81,7 +81,7 @@ type Result struct {
// function to get dsl variables from result struct
func dslVariables() ([]string, error) {
fakeResult := Result{}
fieldsToIgnore := []string{"Hashes", "ResponseHeader"}
fieldsToIgnore := []string{"Hashes", "ResponseHeader", "Err"}
if err := faker.FakeData(&fakeResult, options.WithFieldsToIgnore(fieldsToIgnore...)); err != nil {
return nil, err
}
Expand Down

0 comments on commit 614f997

Please sign in to comment.