Land #79 - stupidly huge refactor

.gitignore

@@ -24,3 +24,5 @@ _testmain.go
 *.prof
 *.txt
 *.swp
+
+gobuster

README.md

@@ -62,6 +62,7 @@ Yes, you're probably correct. Feel free to:
 * `-x <extensions>` - list of extensions to check for, if any.
 * `-P <password>` - HTTP Authorization password (Basic Auth only, prompted if missing).
 * `-U <username>` - HTTP Authorization username (Basic Auth only).
+* `-to <timeout>` - HTTP timeout. Examples: 10s, 100ms, 1m (default: 10s).
 
 ### Building
 
@@ -83,9 +84,9 @@ gobuster$ go run main.go <parameters>
 ```
 
 ### Wordlists via STDIN
-Wordlists can be piped into `gobuster` via stdin:
+Wordlists can be piped into `gobuster` via stdin by providing a `-` to the `-w` option:
 ```
-hashcat -a 3 --stdout ?l | gobuster -u https://mysite.com
+hashcat -a 3 --stdout ?l | gobuster -u https://mysite.com -w -
 ```
 Note: If the `-w` option is specified at the same time as piping from STDIN, an error will be shown and the program will terminate.
 

THANKS

@@ -1,6 +1,6 @@
 @0x42424242 - initial DNS support
 @0xdevalias - Refactoring of code, and lots of other stuff
-@FireFart - Supporting connection reuse, resulting in been speed ups
+@FireFart - Supporting connection reuse, resulting in been speed ups, refactoring, progress output
 @Ne0nd0g - STDIN support for wordlists
 @UID1K - initial DNS wildcard check support
 @averagesecurityguy - quiet mode support
@@ -13,3 +13,4 @@
 @knapsy - saving output to file, and CNAME resolution for DNS mode
 @rverton - CLI flag to skip SSL verification
 @viaMorgoth - base domain validation for DNS mode
+@FireFart - refactoring, progress output

TODO.md

@@ -0,0 +1,3 @@
+* return specific errors and do not mention command line switches in libgobuster
+* no log.Printf and fmt.Printf inside libgobuster
+* use smth like `tabwriter.NewWriter(bw, 0, 5, 3, ' ', 0)` for outputting options (`GetConfigString`)

gobusterdir/gobusterdir.go

@@ -0,0 +1,131 @@
+package gobusterdir
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+
+	"github.com/OJ/gobuster/libgobuster"
+	"github.com/google/uuid"
+)
+
+// GobusterDir is the main type to implement the interface
+type GobusterDir struct{}
+
+// Setup is the setup implementation of gobusterdir
+func (d GobusterDir) Setup(g *libgobuster.Gobuster) error {
+	_, _, err := g.GetRequest(g.Opts.URL)
+	if err != nil {
+		return fmt.Errorf("unable to connect to %s: %v", g.Opts.URL, err)
+	}
+
+	guid := uuid.New()
+	url := fmt.Sprintf("%s%s", g.Opts.URL, guid)
+	wildcardResp, _, err := g.GetRequest(url)
+
+	if err != nil {
+		return err
+	}
+
+	if g.Opts.StatusCodesParsed.Contains(*wildcardResp) {
+		g.IsWildcard = true
+		log.Printf("[-] Wildcard response found: %s => %d", url, *wildcardResp)
+		if !g.Opts.WildcardForced {
+			return fmt.Errorf("[-] To force processing of Wildcard responses, specify the '-fw' switch.")
+		}
+	}
+
+	return nil
+}
+
+// Process is the process implementation of gobusterdir
+func (d GobusterDir) Process(g *libgobuster.Gobuster, word string) ([]libgobuster.Result, error) {
+	suffix := ""
+	if g.Opts.UseSlash {
+		suffix = "/"
+	}
+
+	// Try the DIR first
+	url := fmt.Sprintf("%s%s%s", g.Opts.URL, word, suffix)
+	dirResp, dirSize, err := g.GetRequest(url)
+	if err != nil {
+		return nil, err
+	}
+	var ret []libgobuster.Result
+	if dirResp != nil {
+		ret = append(ret, libgobuster.Result{
+			Entity: fmt.Sprintf("%s%s", word, suffix),
+			Status: *dirResp,
+			Size:   dirSize,
+		})
+	}
+
+	// Follow up with files using each ext.
+	for ext := range g.Opts.ExtensionsParsed.Set {
+		file := fmt.Sprintf("%s.%s", word, ext)
+		url = fmt.Sprintf("%s%s", g.Opts.URL, file)
+		fileResp, fileSize, err := g.GetRequest(url)
+		if err != nil {
+			return nil, err
+		}
+
+		if fileResp != nil {
+			ret = append(ret, libgobuster.Result{
+				Entity: file,
+				Status: *fileResp,
+				Size:   fileSize,
+			})
+		}
+	}
+	return ret, nil
+}
+
+// ResultToString is the to string implementation of gobusterdir
+func (d GobusterDir) ResultToString(g *libgobuster.Gobuster, r *libgobuster.Result) (*string, error) {
+	buf := &bytes.Buffer{}
+
+	// Prefix if we're in verbose mode
+	if g.Opts.Verbose {
+		if g.Opts.StatusCodesParsed.Contains(r.Status) {
+			if _, err := fmt.Fprintf(buf, "Found: "); err != nil {
+				return nil, err
+			}
+		} else {
+			if _, err := fmt.Fprintf(buf, "Missed: "); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	if g.Opts.StatusCodesParsed.Contains(r.Status) || g.Opts.Verbose {
+		if g.Opts.Expanded {
+			if _, err := fmt.Fprintf(buf, g.Opts.URL); err != nil {
+				return nil, err
+			}
+		} else {
+			if _, err := fmt.Fprintf(buf, "/"); err != nil {
+				return nil, err
+			}
+		}
+		if _, err := fmt.Fprintf(buf, r.Entity); err != nil {
+			return nil, err
+		}
+
+		if !g.Opts.NoStatus {
+			if _, err := fmt.Fprintf(buf, " (Status: %d)", r.Status); err != nil {
+				return nil, err
+			}
+		}
+
+		if r.Size != nil {
+			if _, err := fmt.Fprintf(buf, " [Size: %d]", *r.Size); err != nil {
+				return nil, err
+			}
+		}
+		if _, err := fmt.Fprintf(buf, "\n"); err != nil {
+			return nil, err
+		}
+	}
+	s := buf.String()
+	return &s, nil
+}

gobusterdns/gobusterdns.go

@@ -0,0 +1,96 @@
+package gobusterdns
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"net"
+	"strings"
+
+	"github.com/OJ/gobuster/libgobuster"
+	"github.com/google/uuid"
+)
+
+// GobusterDNS is the main type to implement the interface
+type GobusterDNS struct{}
+
+// Setup is the setup implementation of gobusterdns
+func (d GobusterDNS) Setup(g *libgobuster.Gobuster) error {
+	// Resolve a subdomain sthat probably shouldn't exist
+	guid := uuid.New()
+	wildcardIps, err := net.LookupHost(fmt.Sprintf("%s.%s", guid, g.Opts.URL))
+	if err == nil {
+		g.IsWildcard = true
+		g.WildcardIps.AddRange(wildcardIps)
+		log.Printf("[-] Wildcard DNS found. IP address(es): %s", g.WildcardIps.Stringify())
+		if !g.Opts.WildcardForced {
+			return fmt.Errorf("[-] To force processing of Wildcard DNS, specify the '-fw' switch.")
+		}
+	}
+
+	if !g.Opts.Quiet {
+		// Provide a warning if the base domain doesn't resolve (in case of typo)
+		_, err = net.LookupHost(g.Opts.URL)
+		if err != nil {
+			// Not an error, just a warning. Eg. `yp.to` doesn't resolve, but `cr.py.to` does!
+			log.Printf("[-] Unable to validate base domain: %s", g.Opts.URL)
+		}
+	}
+
+	return nil
+}
+
+// Process is the process implementation of gobusterdns
+func (d GobusterDNS) Process(g *libgobuster.Gobuster, word string) ([]libgobuster.Result, error) {
+	subdomain := fmt.Sprintf("%s.%s", word, g.Opts.URL)
+	ips, err := net.LookupHost(subdomain)
+	var ret []libgobuster.Result
+	if err == nil {
+		if !g.IsWildcard || !g.WildcardIps.ContainsAny(ips) {
+			result := libgobuster.Result{
+				Entity: subdomain,
+			}
+			if g.Opts.ShowIPs {
+				result.Extra = strings.Join(ips, ", ")
+			} else if g.Opts.ShowCNAME {
+				cname, err := net.LookupCNAME(subdomain)
+				if err == nil {
+					result.Extra = cname
+				}
+			}
+			ret = append(ret, result)
+		}
+	} else if g.Opts.Verbose {
+		ret = append(ret, libgobuster.Result{
+			Entity: subdomain,
+			Status: 404,
+		})
+	}
+	return ret, nil
+}
+
+// ResultToString is the to string implementation of gobusterdns
+func (d GobusterDNS) ResultToString(g *libgobuster.Gobuster, r *libgobuster.Result) (*string, error) {
+	buf := &bytes.Buffer{}
+
+	if r.Status == 404 {
+		if _, err := fmt.Fprintf(buf, "Missing: %s\n", r.Entity); err != nil {
+			return nil, err
+		}
+	} else if g.Opts.ShowIPs {
+		if _, err := fmt.Fprintf(buf, "Found: %s [%s]\n", r.Entity, r.Extra); err != nil {
+			return nil, err
+		}
+	} else if g.Opts.ShowCNAME {
+		if _, err := fmt.Fprintf(buf, "Found: %s [%s]\n", r.Entity, r.Extra); err != nil {
+			return nil, err
+		}
+	} else {
+		if _, err := fmt.Fprintf(buf, "Found: %s\n", r.Entity); err != nil {
+			return nil, err
+		}
+	}
+
+	s := buf.String()
+	return &s, nil
+}