From d5a25ca2707b0862a70c7d4a87c95bb9fe986e67 Mon Sep 17 00:00:00 2001 From: Christian Mehlmauer Date: Sat, 13 Apr 2019 12:48:26 +0200 Subject: [PATCH 1/5] reduce code reuse --- cli/cmd/dir.go | 157 +++++-------------------- cli/cmd/dns.go | 26 +--- cli/cmd/http.go | 121 +++++++++++++++++++ cli/cmd/root.go | 24 ++++ cli/cmd/vhost.go | 137 +++------------------ go.mod | 6 +- go.sum | 13 +- gobusterdir/helper.go | 39 ------ gobusterdir/options.go | 16 +-- gobustervhost/gobustervhost.go | 6 +- gobustervhost/options.go | 12 +- helper/helper.go | 43 +++++++ {gobusterdir => helper}/helper_test.go | 20 ++-- libgobuster/http.go | 54 +++++++-- libgobuster/options_http.go | 18 +++ 15 files changed, 323 insertions(+), 369 deletions(-) create mode 100644 cli/cmd/http.go delete mode 100644 gobusterdir/helper.go create mode 100644 helper/helper.go rename {gobusterdir => helper}/helper_test.go (80%) create mode 100644 libgobuster/options_http.go diff --git a/cli/cmd/dir.go b/cli/cmd/dir.go index 6cbe3bc8..998d4e32 100644 --- a/cli/cmd/dir.go +++ b/cli/cmd/dir.go @@ -1,22 +1,14 @@ package cmd import ( - "context" "fmt" "log" - "os" - "os/signal" - "regexp" - "strconv" - "strings" - "syscall" - "time" "github.com/OJ/gobuster/v3/cli" "github.com/OJ/gobuster/v3/gobusterdir" + "github.com/OJ/gobuster/v3/helper" "github.com/OJ/gobuster/v3/libgobuster" "github.com/spf13/cobra" - "golang.org/x/crypto/ssh/terminal" ) var cmdDir *cobra.Command @@ -26,32 +18,13 @@ func runDir(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("error on parsing arguments: %v", err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - plugin, err := gobusterdir.NewGobusterDir(ctx, globalopts, pluginopts) + + plugin, err := gobusterdir.NewGobusterDir(mainContext, globalopts, pluginopts) if err != nil { return fmt.Errorf("error on creating gobusterdir: %v", err) } - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, os.Interrupt) - defer func() { - signal.Stop(signalChan) - cancel() - }() - go func() { - select { - case <-signalChan: - // caught CTRL+C - if !globalopts.Quiet { - fmt.Println("\n[!] Keyboard interrupt detected, terminating.") - } - cancel() - case <-ctx.Done(): - } - }() - - if err := cli.Gobuster(ctx, globalopts, plugin); err != nil { + if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { return fmt.Errorf("error on running goubster: %v", err) } return nil @@ -65,54 +38,19 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { plugin := gobusterdir.NewOptionsDir() - plugin.URL, err = cmdDir.Flags().GetString("url") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for url: %v", err) - } - - if !strings.HasPrefix(plugin.URL, "http") { - // check to see if a port was specified - re := regexp.MustCompile(`^[^/]+:(\d+)`) - match := re.FindStringSubmatch(plugin.URL) - - if len(match) < 2 { - // no port, default to http on 80 - plugin.URL = fmt.Sprintf("http://%s", plugin.URL) - } else { - port, err2 := strconv.Atoi(match[1]) - if err2 != nil || (port != 80 && port != 443) { - return nil, nil, fmt.Errorf("url scheme not specified") - } else if port == 80 { - plugin.URL = fmt.Sprintf("http://%s", plugin.URL) - } else { - plugin.URL = fmt.Sprintf("https://%s", plugin.URL) - } - } - } - - plugin.StatusCodes, err = cmdDir.Flags().GetString("statuscodes") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for statuscodes: %v", err) - } - - if err = plugin.ParseStatusCodes(); err != nil { - return nil, nil, fmt.Errorf("invalid value for statuscodes: %v", err) - } - - plugin.Cookies, err = cmdDir.Flags().GetString("cookies") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for cookies: %v", err) - } - - plugin.Username, err = cmdDir.Flags().GetString("username") + httpOpts, err := parseCommonHTTPOptions(cmdDir) if err != nil { - return nil, nil, fmt.Errorf("invalid value for username: %v", err) - } - - plugin.Password, err = cmdDir.Flags().GetString("password") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for password: %v", err) + return nil, nil, err } + plugin.Password = httpOpts.Password + plugin.URL = httpOpts.URL + plugin.UserAgent = httpOpts.UserAgent + plugin.Username = httpOpts.Username + plugin.Proxy = httpOpts.Proxy + plugin.Cookies = httpOpts.Cookies + plugin.Timeout = httpOpts.Timeout + plugin.FollowRedirect = httpOpts.FollowRedirect + plugin.InsecureSSL = httpOpts.InsecureSSL plugin.Extensions, err = cmdDir.Flags().GetString("extensions") if err != nil { @@ -120,29 +58,27 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { } if plugin.Extensions != "" { - if err = plugin.ParseExtensions(); err != nil { + ret, err := helper.ParseExtensions(plugin.Extensions) + if err != nil { return nil, nil, fmt.Errorf("invalid value for extensions: %v", err) } + plugin.ExtensionsParsed = *ret } - plugin.UserAgent, err = cmdDir.Flags().GetString("useragent") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for useragent: %v", err) - } - - plugin.Proxy, err = cmdDir.Flags().GetString("proxy") + plugin.StatusCodes, err = cmdDir.Flags().GetString("statuscodes") if err != nil { - return nil, nil, fmt.Errorf("invalid value for proxy: %v", err) + return nil, nil, fmt.Errorf("invalid value for statuscodes: %v", err) } - plugin.Timeout, err = cmdDir.Flags().GetDuration("timeout") + ret, err := helper.ParseStatusCodes(plugin.StatusCodes) if err != nil { - return nil, nil, fmt.Errorf("invalid value for timeout: %v", err) + return nil, nil, fmt.Errorf("invalid value for statuscodes: %v", err) } + plugin.StatusCodesParsed = *ret - plugin.FollowRedirect, err = cmdDir.Flags().GetBool("followredirect") + plugin.UseSlash, err = cmdDir.Flags().GetBool("addslash") if err != nil { - return nil, nil, fmt.Errorf("invalid value for followredirect: %v", err) + return nil, nil, fmt.Errorf("invalid value for addslash: %v", err) } plugin.Expanded, err = cmdDir.Flags().GetBool("expanded") @@ -160,38 +96,11 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { return nil, nil, fmt.Errorf("invalid value for includelength: %v", err) } - plugin.InsecureSSL, err = cmdDir.Flags().GetBool("insecuressl") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for insecuressl: %v", err) - } - plugin.WildcardForced, err = cmdDir.Flags().GetBool("wildcard") if err != nil { return nil, nil, fmt.Errorf("invalid value for wildcard: %v", err) } - plugin.UseSlash, err = cmdDir.Flags().GetBool("addslash") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for addslash: %v", err) - } - - // Prompt for PW if not provided - if plugin.Username != "" && plugin.Password == "" { - fmt.Printf("[?] Auth Password: ") - passBytes, err := terminal.ReadPassword(int(syscall.Stdin)) - // print a newline to simulate the newline that was entered - // this means that formatting/printing after doesn't look bad. - fmt.Println("") - if err != nil { - return nil, nil, fmt.Errorf("username given but reading of password failed") - } - plugin.Password = string(passBytes) - } - // if it's still empty bail out - if plugin.Username != "" && plugin.Password == "" { - return nil, nil, fmt.Errorf("username was provided but password is missing") - } - return globalopts, plugin, nil } @@ -202,27 +111,17 @@ func init() { RunE: runDir, } - cmdDir.Flags().StringP("url", "u", "", "The target URL") + if err := addCommonHTTPOptions(cmdDir); err != nil { + log.Fatalf("%v", err) + } cmdDir.Flags().StringP("statuscodes", "s", "200,204,301,302,307,401,403", "Positive status codes") - cmdDir.Flags().StringP("cookies", "c", "", "Cookies to use for the requests") - cmdDir.Flags().StringP("username", "U", "", "Username for Basic Auth") - cmdDir.Flags().StringP("password", "P", "", "Password for Basic Auth") cmdDir.Flags().StringP("extensions", "x", "", "File extension(s) to search for") - cmdDir.Flags().StringP("useragent", "a", libgobuster.DefaultUserAgent(), "Set the User-Agent string") - cmdDir.Flags().StringP("proxy", "p", "", "Proxy to use for requests [http(s)://host:port]") - cmdDir.Flags().DurationP("timeout", "", 10*time.Second, "HTTP Timeout") - cmdDir.Flags().BoolP("followredirect", "r", false, "Follow redirects") cmdDir.Flags().BoolP("expanded", "e", false, "Expanded mode, print full URLs") cmdDir.Flags().BoolP("nostatus", "n", false, "Don't print status codes") cmdDir.Flags().BoolP("includelength", "l", false, "Include the length of the body in the output") - cmdDir.Flags().BoolP("insecuressl", "k", false, "Skip SSL certificate verification") cmdDir.Flags().BoolP("addslash", "f", false, "Apped / to each request") cmdDir.Flags().BoolP("wildcard", "", false, "Force continued operation when wildcard found") - if err := cmdDir.MarkFlagRequired("url"); err != nil { - log.Fatalf("error on marking flag as required: %v", err) - } - cmdDir.PersistentPreRun = func(cmd *cobra.Command, args []string) { configureGlobalOptions() } diff --git a/cli/cmd/dns.go b/cli/cmd/dns.go index 0cb2ac77..7241c0b6 100644 --- a/cli/cmd/dns.go +++ b/cli/cmd/dns.go @@ -1,11 +1,8 @@ package cmd import ( - "context" "fmt" "log" - "os" - "os/signal" "runtime" "time" @@ -22,32 +19,13 @@ func runDNS(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("error on parsing arguments: %v", err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + plugin, err := gobusterdns.NewGobusterDNS(globalopts, pluginopts) if err != nil { return fmt.Errorf("Error on creating gobusterdns: %v", err) } - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, os.Interrupt) - defer func() { - signal.Stop(signalChan) - cancel() - }() - go func() { - select { - case <-signalChan: - // caught CTRL+C - if !globalopts.Quiet { - fmt.Println("\n[!] Keyboard interrupt detected, terminating.") - } - cancel() - case <-ctx.Done(): - } - }() - - if err := cli.Gobuster(ctx, globalopts, plugin); err != nil { + if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { return fmt.Errorf("error on running goubster: %v", err) } return nil diff --git a/cli/cmd/http.go b/cli/cmd/http.go new file mode 100644 index 00000000..9001c603 --- /dev/null +++ b/cli/cmd/http.go @@ -0,0 +1,121 @@ +package cmd + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "github.com/OJ/gobuster/v3/libgobuster" + "github.com/spf13/cobra" + "golang.org/x/crypto/ssh/terminal" +) + +func addCommonHTTPOptions(cmd *cobra.Command) error { + cmd.Flags().StringP("url", "u", "", "The target URL") + cmd.Flags().StringP("cookies", "c", "", "Cookies to use for the requests") + cmd.Flags().StringP("username", "U", "", "Username for Basic Auth") + cmd.Flags().StringP("password", "P", "", "Password for Basic Auth") + cmd.Flags().StringP("useragent", "a", libgobuster.DefaultUserAgent(), "Set the User-Agent string") + cmd.Flags().StringP("proxy", "p", "", "Proxy to use for requests [http(s)://host:port]") + cmd.Flags().DurationP("timeout", "", 10*time.Second, "HTTP Timeout") + cmd.Flags().BoolP("followredirect", "r", false, "Follow redirects") + cmd.Flags().BoolP("insecuressl", "k", false, "Skip SSL certificate verification") + + if err := cmdDir.MarkFlagRequired("url"); err != nil { + return fmt.Errorf("error on marking flag as required: %v", err) + } + + return nil +} + +func parseCommonHTTPOptions(cmd *cobra.Command) (libgobuster.OptionsHTTP, error) { + options := libgobuster.OptionsHTTP{} + var err error + + options.URL, err = cmd.Flags().GetString("url") + if err != nil { + return options, fmt.Errorf("invalid value for url: %v", err) + } + + if !strings.HasPrefix(options.URL, "http") { + // check to see if a port was specified + re := regexp.MustCompile(`^[^/]+:(\d+)`) + match := re.FindStringSubmatch(options.URL) + + if len(match) < 2 { + // no port, default to http on 80 + options.URL = fmt.Sprintf("http://%s", options.URL) + } else { + port, err2 := strconv.Atoi(match[1]) + if err2 != nil || (port != 80 && port != 443) { + return options, fmt.Errorf("url scheme not specified") + } else if port == 80 { + options.URL = fmt.Sprintf("http://%s", options.URL) + } else { + options.URL = fmt.Sprintf("https://%s", options.URL) + } + } + } + + options.Cookies, err = cmd.Flags().GetString("cookies") + if err != nil { + return options, fmt.Errorf("invalid value for cookies: %v", err) + } + + options.Username, err = cmd.Flags().GetString("username") + if err != nil { + return options, fmt.Errorf("invalid value for username: %v", err) + } + + options.Password, err = cmd.Flags().GetString("password") + if err != nil { + return options, fmt.Errorf("invalid value for password: %v", err) + } + + options.UserAgent, err = cmd.Flags().GetString("useragent") + if err != nil { + return options, fmt.Errorf("invalid value for useragent: %v", err) + } + + options.Proxy, err = cmd.Flags().GetString("proxy") + if err != nil { + return options, fmt.Errorf("invalid value for proxy: %v", err) + } + + options.Timeout, err = cmd.Flags().GetDuration("timeout") + if err != nil { + return options, fmt.Errorf("invalid value for timeout: %v", err) + } + + options.FollowRedirect, err = cmd.Flags().GetBool("followredirect") + if err != nil { + return options, fmt.Errorf("invalid value for followredirect: %v", err) + } + + options.InsecureSSL, err = cmd.Flags().GetBool("insecuressl") + if err != nil { + return options, fmt.Errorf("invalid value for insecuressl: %v", err) + } + + // Prompt for PW if not provided + if options.Username != "" && options.Password == "" { + fmt.Printf("[?] Auth Password: ") + passBytes, err := terminal.ReadPassword(int(syscall.Stdin)) + // print a newline to simulate the newline that was entered + // this means that formatting/printing after doesn't look bad. + fmt.Println("") + if err != nil { + return options, fmt.Errorf("username given but reading of password failed") + } + options.Password = string(passBytes) + } + // if it's still empty bail out + if options.Username != "" && options.Password == "" { + return options, fmt.Errorf("username was provided but password is missing") + } + + return options, nil +} diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 273c3381..bf18c98d 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -1,9 +1,11 @@ package cmd import ( + "context" "fmt" "log" "os" + "os/signal" "github.com/OJ/gobuster/v3/libgobuster" @@ -14,8 +16,30 @@ var rootCmd = &cobra.Command{ Use: "gobuster", } +var mainContext context.Context + // Execute is the main cobra method func Execute() { + var cancel context.CancelFunc + mainContext, cancel = context.WithCancel(context.Background()) + defer cancel() + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt) + defer func() { + signal.Stop(signalChan) + cancel() + }() + go func() { + select { + case <-signalChan: + // caught CTRL+C + fmt.Println("\n[!] Keyboard interrupt detected, terminating.") + cancel() + case <-mainContext.Done(): + } + }() + if err := rootCmd.Execute(); err != nil { // Leaving this in results in the same error appearing twice // Once before and once after the help output. Not sure if diff --git a/cli/cmd/vhost.go b/cli/cmd/vhost.go index 907e3591..db96a081 100644 --- a/cli/cmd/vhost.go +++ b/cli/cmd/vhost.go @@ -1,22 +1,13 @@ package cmd import ( - "context" "fmt" "log" - "os" - "os/signal" - "regexp" - "strconv" - "strings" - "syscall" - "time" "github.com/OJ/gobuster/v3/cli" "github.com/OJ/gobuster/v3/gobustervhost" "github.com/OJ/gobuster/v3/libgobuster" "github.com/spf13/cobra" - "golang.org/x/crypto/ssh/terminal" ) var cmdVhost *cobra.Command @@ -26,32 +17,13 @@ func runVhost(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("error on parsing arguments: %v", err) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - plugin, err := gobustervhost.NewGobusterVhost(ctx, globalopts, pluginopts) + + plugin, err := gobustervhost.NewGobusterVhost(mainContext, globalopts, pluginopts) if err != nil { return fmt.Errorf("error on creating gobustervhost: %v", err) } - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, os.Interrupt) - defer func() { - signal.Stop(signalChan) - cancel() - }() - go func() { - select { - case <-signalChan: - // caught CTRL+C - if !globalopts.Quiet { - fmt.Println("\n[!] Keyboard interrupt detected, terminating.") - } - cancel() - case <-ctx.Done(): - } - }() - - if err := cli.Gobuster(ctx, globalopts, plugin); err != nil { + if err := cli.Gobuster(mainContext, globalopts, plugin); err != nil { return fmt.Errorf("error on running goubster: %v", err) } return nil @@ -64,87 +36,19 @@ func parseVhostOptions() (*libgobuster.Options, *gobustervhost.OptionsVhost, err } var plugin gobustervhost.OptionsVhost - url, err := cmdVhost.Flags().GetString("url") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for url: %v", err) - } - plugin.URL = url - if !strings.HasPrefix(plugin.URL, "http") { - // check to see if a port was specified - re := regexp.MustCompile(`^[^/]+:(\d+)`) - match := re.FindStringSubmatch(plugin.URL) - - if len(match) < 2 { - // no port, default to http on 80 - plugin.URL = fmt.Sprintf("http://%s", plugin.URL) - } else { - port, err2 := strconv.Atoi(match[1]) - if err2 != nil || (port != 80 && port != 443) { - return nil, nil, fmt.Errorf("url scheme not specified") - } else if port == 80 { - plugin.URL = fmt.Sprintf("http://%s", plugin.URL) - } else { - plugin.URL = fmt.Sprintf("https://%s", plugin.URL) - } - } - } - - plugin.Cookies, err = cmdVhost.Flags().GetString("cookies") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for cookies: %v", err) - } - - plugin.Username, err = cmdVhost.Flags().GetString("username") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for username: %v", err) - } - - plugin.Password, err = cmdVhost.Flags().GetString("password") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for password: %v", err) - } - - plugin.UserAgent, err = cmdVhost.Flags().GetString("useragent") + httpOpts, err := parseCommonHTTPOptions(cmdVhost) if err != nil { - return nil, nil, fmt.Errorf("invalid value for useragent: %v", err) - } - - plugin.Proxy, err = cmdVhost.Flags().GetString("proxy") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for proxy: %v", err) - } - - plugin.Timeout, err = cmdVhost.Flags().GetDuration("timeout") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for timeout: %v", err) - } - - plugin.FollowRedirect, err = cmdDir.Flags().GetBool("followredirect") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for followredirect: %v", err) - } - - plugin.InsecureSSL, err = cmdVhost.Flags().GetBool("insecuressl") - if err != nil { - return nil, nil, fmt.Errorf("invalid value for insecuressl: %v", err) - } - - // Prompt for PW if not provided - if plugin.Username != "" && plugin.Password == "" { - fmt.Printf("[?] Auth Password: ") - passBytes, err := terminal.ReadPassword(int(syscall.Stdin)) - // print a newline to simulate the newline that was entered - // this means that formatting/printing after doesn't look bad. - fmt.Println("") - if err != nil { - return nil, nil, fmt.Errorf("username given but reading of password failed") - } - plugin.Password = string(passBytes) - } - // if it's still empty bail out - if plugin.Username != "" && plugin.Password == "" { - return nil, nil, fmt.Errorf("username was provided but password is missing") + return nil, nil, err } + plugin.Password = httpOpts.Password + plugin.URL = httpOpts.URL + plugin.UserAgent = httpOpts.UserAgent + plugin.Username = httpOpts.Username + plugin.Proxy = httpOpts.Proxy + plugin.Cookies = httpOpts.Cookies + plugin.Timeout = httpOpts.Timeout + plugin.FollowRedirect = httpOpts.FollowRedirect + plugin.InsecureSSL = httpOpts.InsecureSSL return globalopts, &plugin, nil } @@ -155,17 +59,8 @@ func init() { Short: "Uses VHOST bruteforcing mode", RunE: runVhost, } - cmdVhost.Flags().StringP("url", "u", "", "The target URL") - cmdVhost.Flags().StringP("cookies", "c", "", "Cookies to use for the requests") - cmdVhost.Flags().StringP("username", "U", "", "Username for Basic Auth") - cmdVhost.Flags().StringP("password", "P", "", "Password for Basic Auth") - cmdVhost.Flags().StringP("useragent", "a", libgobuster.DefaultUserAgent(), "Set the User-Agent string") - cmdVhost.Flags().StringP("proxy", "p", "", "Proxy to use for requests [http(s)://host:port]") - cmdVhost.Flags().DurationP("timeout", "", 10*time.Second, "HTTP Timeout") - cmdVhost.Flags().BoolP("followredirect", "r", true, "Follow redirects") - cmdVhost.Flags().BoolP("insecuressl", "k", false, "Skip SSL certificate verification") - if err := cmdVhost.MarkFlagRequired("url"); err != nil { - log.Fatalf("error on marking flag as required: %v", err) + if err := addCommonHTTPOptions(cmdVhost); err != nil { + log.Fatalf("%v", err) } cmdVhost.PersistentPreRun = func(cmd *cobra.Command, args []string) { diff --git a/go.mod b/go.mod index 95c1135f..f73943a8 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/OJ/gobuster/v3 require ( - github.com/google/uuid v1.1.0 + github.com/google/uuid v1.1.1 github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect - golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc - golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb // indirect + golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a + golang.org/x/sys v0.0.0-20190412213103-97732733099d // indirect ) diff --git a/go.sum b/go.sum index fdcab092..164d9c06 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,13 @@ -github.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s= -github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk= -golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a h1:Igim7XhdOpBnWPuYJ70XcNpq8q3BCACtVgNfoJxOV7g= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/gobusterdir/helper.go b/gobusterdir/helper.go deleted file mode 100644 index 3c10682b..00000000 --- a/gobusterdir/helper.go +++ /dev/null @@ -1,39 +0,0 @@ -package gobusterdir - -import ( - "fmt" - "strconv" - "strings" -) - -// ParseExtensions parses the extensions provided as a comma seperated list -func (opt *OptionsDir) ParseExtensions() error { - if opt.Extensions == "" { - return fmt.Errorf("invalid extension string provided") - } - - exts := strings.Split(opt.Extensions, ",") - for _, e := range exts { - e = strings.TrimSpace(e) - // remove leading . from extensions - opt.ExtensionsParsed.Add(strings.TrimPrefix(e, ".")) - } - return nil -} - -// ParseStatusCodes parses the status codes provided as a comma seperated list -func (opt *OptionsDir) ParseStatusCodes() error { - if opt.StatusCodes == "" { - return fmt.Errorf("invalid status code string provided") - } - - for _, c := range strings.Split(opt.StatusCodes, ",") { - c = strings.TrimSpace(c) - i, err := strconv.Atoi(c) - if err != nil { - return fmt.Errorf("invalid status code given: %s", c) - } - opt.StatusCodesParsed.Add(i) - } - return nil -} diff --git a/gobusterdir/options.go b/gobusterdir/options.go index f039b66e..1e11f03d 100644 --- a/gobusterdir/options.go +++ b/gobusterdir/options.go @@ -1,31 +1,21 @@ package gobusterdir import ( - "time" - "github.com/OJ/gobuster/v3/libgobuster" ) // OptionsDir is the struct to hold all options for this plugin type OptionsDir struct { + libgobuster.OptionsHTTP Extensions string ExtensionsParsed libgobuster.StringSet - Password string StatusCodes string StatusCodesParsed libgobuster.IntSet - URL string - UserAgent string - Username string - Proxy string - Cookies string - Timeout time.Duration - FollowRedirect bool + UseSlash bool + WildcardForced bool IncludeLength bool Expanded bool NoStatus bool - InsecureSSL bool - UseSlash bool - WildcardForced bool } // NewOptionsDir returns a new initialized OptionsDir diff --git a/gobustervhost/gobustervhost.go b/gobustervhost/gobustervhost.go index 259289ea..79f95b36 100644 --- a/gobustervhost/gobustervhost.go +++ b/gobustervhost/gobustervhost.go @@ -71,7 +71,7 @@ func (v *GobusterVhost) PreRun() error { v.domain = url.Host // request default vhost for baseline1 - _, tmp, err := v.http.GetBody(v.options.URL, "", v.options.Cookies) + _, tmp, err := v.http.GetWithBody(v.options.URL, "", v.options.Cookies) if err != nil { return fmt.Errorf("unable to connect to %s: %v", v.options.URL, err) } @@ -79,7 +79,7 @@ func (v *GobusterVhost) PreRun() error { // request non existent vhost for baseline2 subdomain := fmt.Sprintf("%s.%s", uuid.New(), v.domain) - _, tmp, err = v.http.GetBody(v.options.URL, subdomain, v.options.Cookies) + _, tmp, err = v.http.GetWithBody(v.options.URL, subdomain, v.options.Cookies) if err != nil { return fmt.Errorf("unable to connect to %s: %v", v.options.URL, err) } @@ -90,7 +90,7 @@ func (v *GobusterVhost) PreRun() error { // Run is the process implementation of gobusterdir func (v *GobusterVhost) Run(word string) ([]libgobuster.Result, error) { subdomain := fmt.Sprintf("%s.%s", word, v.domain) - status, body, err := v.http.GetBody(v.options.URL, subdomain, v.options.Cookies) + status, body, err := v.http.GetWithBody(v.options.URL, subdomain, v.options.Cookies) var ret []libgobuster.Result if err != nil { return ret, err diff --git a/gobustervhost/options.go b/gobustervhost/options.go index 25e97728..baefb080 100644 --- a/gobustervhost/options.go +++ b/gobustervhost/options.go @@ -1,18 +1,10 @@ package gobustervhost import ( - "time" + "github.com/OJ/gobuster/v3/libgobuster" ) // OptionsVhost is the struct to hold all options for this plugin type OptionsVhost struct { - Password string - URL string - UserAgent string - Username string - Proxy string - Cookies string - Timeout time.Duration - InsecureSSL bool - FollowRedirect bool + libgobuster.OptionsHTTP } diff --git a/helper/helper.go b/helper/helper.go new file mode 100644 index 00000000..028dc60b --- /dev/null +++ b/helper/helper.go @@ -0,0 +1,43 @@ +package helper + +import ( + "fmt" + "strconv" + "strings" + + "github.com/OJ/gobuster/v3/libgobuster" +) + +// ParseExtensions parses the extensions provided as a comma seperated list +func ParseExtensions(extensions string) (*libgobuster.StringSet, error) { + if extensions == "" { + return nil, fmt.Errorf("invalid extension string provided") + } + + ret := libgobuster.NewStringSet() + exts := strings.Split(extensions, ",") + for _, e := range exts { + e = strings.TrimSpace(e) + // remove leading . from extensions + ret.Add(strings.TrimPrefix(e, ".")) + } + return &ret, nil +} + +// ParseStatusCodes parses the status codes provided as a comma seperated list +func ParseStatusCodes(statuscodes string) (*libgobuster.IntSet, error) { + if statuscodes == "" { + return nil, fmt.Errorf("invalid status code string provided") + } + + ret := libgobuster.NewIntSet() + for _, c := range strings.Split(statuscodes, ",") { + c = strings.TrimSpace(c) + i, err := strconv.Atoi(c) + if err != nil { + return nil, fmt.Errorf("invalid status code given: %s", c) + } + ret.Add(i) + } + return &ret, nil +} diff --git a/gobusterdir/helper_test.go b/helper/helper_test.go similarity index 80% rename from gobusterdir/helper_test.go rename to helper/helper_test.go index b81acc32..447b8719 100644 --- a/gobusterdir/helper_test.go +++ b/helper/helper_test.go @@ -1,4 +1,4 @@ -package gobusterdir +package helper import ( "reflect" @@ -12,7 +12,7 @@ func TestParseExtensions(t *testing.T) { var tt = []struct { testName string - Extensions string + extensions string expectedExtensions libgobuster.StringSet expectedError string }{ @@ -25,15 +25,13 @@ func TestParseExtensions(t *testing.T) { for _, x := range tt { t.Run(x.testName, func(t *testing.T) { - o := NewOptionsDir() - o.Extensions = x.Extensions - err := o.ParseExtensions() + ret, err := ParseExtensions(x.extensions) if x.expectedError != "" { if err.Error() != x.expectedError { t.Fatalf("Expected error %q but got %q", x.expectedError, err.Error()) } - } else if !reflect.DeepEqual(x.expectedExtensions, o.ExtensionsParsed) { - t.Fatalf("Expected %v but got %v", x.expectedExtensions, o.ExtensionsParsed) + } else if !reflect.DeepEqual(x.expectedExtensions, ret) { + t.Fatalf("Expected %v but got %v", x.expectedExtensions, ret) } }) } @@ -58,15 +56,13 @@ func TestParseStatusCodes(t *testing.T) { for _, x := range tt { t.Run(x.testName, func(t *testing.T) { - o := NewOptionsDir() - o.StatusCodes = x.stringCodes - err := o.ParseStatusCodes() + ret, err := ParseStatusCodes(x.stringCodes) if x.expectedError != "" { if err.Error() != x.expectedError { t.Fatalf("Expected error %q but got %q", x.expectedError, err.Error()) } - } else if !reflect.DeepEqual(x.expectedCodes, o.StatusCodesParsed) { - t.Fatalf("Expected %v but got %v", x.expectedCodes, o.StatusCodesParsed) + } else if !reflect.DeepEqual(x.expectedCodes, ret) { + t.Fatalf("Expected %v but got %v", x.expectedCodes, ret) } }) } diff --git a/libgobuster/http.go b/libgobuster/http.go index b9016f0a..a57d2195 100644 --- a/libgobuster/http.go +++ b/libgobuster/http.go @@ -1,6 +1,7 @@ package libgobuster import ( + "bytes" "context" "crypto/tls" "fmt" @@ -79,9 +80,29 @@ func NewHTTPClient(c context.Context, opt *HTTPOptions) (*HTTPClient, error) { return &client, nil } -// Get makes an http request and returns the status, the length and an error +// Get gets an URL and returns the status, the length and an error func (client *HTTPClient) Get(fullURL, host, cookie string) (*int, *int64, error) { - resp, err := client.makeRequest(fullURL, host, cookie) + return client.requestWithoutBody(http.MethodGet, fullURL, host, cookie, "") +} + +// Post posts to an URL and returns the status, the length and an error +func (client *HTTPClient) Post(fullURL, host, cookie, data string) (*int, *int64, error) { + return client.requestWithoutBody(http.MethodPost, fullURL, host, cookie, data) +} + +// GetWithBody gets an URL and returns the status and the body +func (client *HTTPClient) GetWithBody(fullURL, host, cookie string) (*int, *string, error) { + return client.requestWithBody(http.MethodGet, fullURL, host, cookie, "") +} + +// PostWithBody gets an URL and returns the status and the body +func (client *HTTPClient) PostWithBody(fullURL, host, cookie, data string) (*int, *string, error) { + return client.requestWithBody(http.MethodPost, fullURL, host, cookie, data) +} + +// requestWithoutBody makes an http request and returns the status, the length and an error +func (client *HTTPClient) requestWithoutBody(method, fullURL, host, cookie, data string) (*int, *int64, error) { + resp, err := client.makeRequest(method, fullURL, host, cookie, data) if err != nil { // ignore context canceled errors if client.context.Err() == context.Canceled { @@ -115,9 +136,9 @@ func (client *HTTPClient) Get(fullURL, host, cookie string) (*int, *int64, error return &resp.StatusCode, length, nil } -// GetBody makes an http request and returns the status and the body -func (client *HTTPClient) GetBody(fullURL, host, cookie string) (*int, *string, error) { - resp, err := client.makeRequest(fullURL, host, cookie) +// requestWithBody makes an http request and returns the status and the body +func (client *HTTPClient) requestWithBody(method, fullURL, host, cookie, data string) (*int, *string, error) { + resp, err := client.makeRequest(method, fullURL, host, cookie, data) if err != nil { // ignore context canceled errors if client.context.Err() == context.Canceled { @@ -136,10 +157,24 @@ func (client *HTTPClient) GetBody(fullURL, host, cookie string) (*int, *string, return &resp.StatusCode, &bodyString, nil } -func (client *HTTPClient) makeRequest(fullURL, host, cookie string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodGet, fullURL, nil) - if err != nil { - return nil, err +func (client *HTTPClient) makeRequest(method, fullURL, host, cookie, data string) (*http.Response, error) { + var req *http.Request + var err error + + switch method { + case http.MethodGet: + req, err = http.NewRequest(http.MethodGet, fullURL, nil) + if err != nil { + return nil, err + } + case http.MethodPost: + buf := bytes.NewBufferString(data) + req, err = http.NewRequest(http.MethodPost, fullURL, buf) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid method %s", method) } // add the context so we can easily cancel out @@ -172,5 +207,6 @@ func (client *HTTPClient) makeRequest(fullURL, host, cookie string) (*http.Respo } return nil, err } + return resp, nil } diff --git a/libgobuster/options_http.go b/libgobuster/options_http.go new file mode 100644 index 00000000..747634d6 --- /dev/null +++ b/libgobuster/options_http.go @@ -0,0 +1,18 @@ +package libgobuster + +import ( + "time" +) + +// OptionsHTTP is the struct to hold all options for common HTTP options +type OptionsHTTP struct { + Password string + URL string + UserAgent string + Username string + Proxy string + Cookies string + Timeout time.Duration + FollowRedirect bool + InsecureSSL bool +} From 4e7f510c9853629492db8404291c22ea37d2dd1c Mon Sep 17 00:00:00 2001 From: Christian Mehlmauer Date: Tue, 16 Apr 2019 23:44:58 +0200 Subject: [PATCH 2/5] more optimization and benchmark --- .gitignore | 1 + cli/cmd/dir.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- gobustervhost/gobustervhost.go | 6 ++--- helper/helper.go | 14 ++++++------ libgobuster/http.go | 29 +++++++++++------------ libgobuster/http_test.go | 42 +++++++++++++++++++++++++++++++--- 8 files changed, 70 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index cd146d8c..46167de3 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ _testmain.go gobuster build +v3 diff --git a/cli/cmd/dir.go b/cli/cmd/dir.go index 998d4e32..58023ed0 100644 --- a/cli/cmd/dir.go +++ b/cli/cmd/dir.go @@ -62,7 +62,7 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { if err != nil { return nil, nil, fmt.Errorf("invalid value for extensions: %v", err) } - plugin.ExtensionsParsed = *ret + plugin.ExtensionsParsed = ret } plugin.StatusCodes, err = cmdDir.Flags().GetString("statuscodes") @@ -74,7 +74,7 @@ func parseDirOptions() (*libgobuster.Options, *gobusterdir.OptionsDir, error) { if err != nil { return nil, nil, fmt.Errorf("invalid value for statuscodes: %v", err) } - plugin.StatusCodesParsed = *ret + plugin.StatusCodesParsed = ret plugin.UseSlash, err = cmdDir.Flags().GetBool("addslash") if err != nil { diff --git a/go.mod b/go.mod index f73943a8..8a4a74ce 100644 --- a/go.mod +++ b/go.mod @@ -6,5 +6,5 @@ require ( github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 // indirect golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a - golang.org/x/sys v0.0.0-20190412213103-97732733099d // indirect + golang.org/x/sys v0.0.0-20190416152802-12500544f89f // indirect ) diff --git a/go.sum b/go.sum index 164d9c06..9a380269 100644 --- a/go.sum +++ b/go.sum @@ -9,5 +9,5 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a h1:Igim7XhdOpBnWPuYJ70XcNpq8q3BCACtVgNfoJxOV7g= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190416152802-12500544f89f h1:1ZH9RnjNgLzh6YrsRp/c6ddZ8Lq0fq9xztNOoWJ2sz4= +golang.org/x/sys v0.0.0-20190416152802-12500544f89f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/gobustervhost/gobustervhost.go b/gobustervhost/gobustervhost.go index 79f95b36..68600189 100644 --- a/gobustervhost/gobustervhost.go +++ b/gobustervhost/gobustervhost.go @@ -19,8 +19,8 @@ type GobusterVhost struct { globalopts *libgobuster.Options http *libgobuster.HTTPClient domain string - baseline1 string - baseline2 string + baseline1 []byte + baseline2 []byte } // NewGobusterVhost creates a new initialized GobusterDir @@ -98,7 +98,7 @@ func (v *GobusterVhost) Run(word string) ([]libgobuster.Result, error) { // subdomain must not match default vhost and non existent vhost // or verbose mode is enabled - found := *body != v.baseline1 && *body != v.baseline2 + found := !bytes.Equal(*body, v.baseline1) && !bytes.Equal(*body, v.baseline2) if found || v.globalopts.Verbose { size := int64(len(*body)) resultStatus := libgobuster.StatusMissed diff --git a/helper/helper.go b/helper/helper.go index 028dc60b..a665e1b9 100644 --- a/helper/helper.go +++ b/helper/helper.go @@ -9,9 +9,9 @@ import ( ) // ParseExtensions parses the extensions provided as a comma seperated list -func ParseExtensions(extensions string) (*libgobuster.StringSet, error) { +func ParseExtensions(extensions string) (libgobuster.StringSet, error) { if extensions == "" { - return nil, fmt.Errorf("invalid extension string provided") + return libgobuster.StringSet{}, fmt.Errorf("invalid extension string provided") } ret := libgobuster.NewStringSet() @@ -21,13 +21,13 @@ func ParseExtensions(extensions string) (*libgobuster.StringSet, error) { // remove leading . from extensions ret.Add(strings.TrimPrefix(e, ".")) } - return &ret, nil + return ret, nil } // ParseStatusCodes parses the status codes provided as a comma seperated list -func ParseStatusCodes(statuscodes string) (*libgobuster.IntSet, error) { +func ParseStatusCodes(statuscodes string) (libgobuster.IntSet, error) { if statuscodes == "" { - return nil, fmt.Errorf("invalid status code string provided") + return libgobuster.IntSet{}, fmt.Errorf("invalid status code string provided") } ret := libgobuster.NewIntSet() @@ -35,9 +35,9 @@ func ParseStatusCodes(statuscodes string) (*libgobuster.IntSet, error) { c = strings.TrimSpace(c) i, err := strconv.Atoi(c) if err != nil { - return nil, fmt.Errorf("invalid status code given: %s", c) + return libgobuster.IntSet{}, fmt.Errorf("invalid status code given: %s", c) } ret.Add(i) } - return &ret, nil + return ret, nil } diff --git a/libgobuster/http.go b/libgobuster/http.go index a57d2195..5eef1950 100644 --- a/libgobuster/http.go +++ b/libgobuster/http.go @@ -14,6 +14,8 @@ import ( "unicode/utf8" ) +var defaultUserAgent = DefaultUserAgent() + // HTTPClient represents a http object type HTTPClient struct { client *http.Client @@ -82,26 +84,26 @@ func NewHTTPClient(c context.Context, opt *HTTPOptions) (*HTTPClient, error) { // Get gets an URL and returns the status, the length and an error func (client *HTTPClient) Get(fullURL, host, cookie string) (*int, *int64, error) { - return client.requestWithoutBody(http.MethodGet, fullURL, host, cookie, "") + return client.requestWithoutBody(http.MethodGet, fullURL, host, cookie, nil) } // Post posts to an URL and returns the status, the length and an error -func (client *HTTPClient) Post(fullURL, host, cookie, data string) (*int, *int64, error) { +func (client *HTTPClient) Post(fullURL, host, cookie string, data []byte) (*int, *int64, error) { return client.requestWithoutBody(http.MethodPost, fullURL, host, cookie, data) } // GetWithBody gets an URL and returns the status and the body -func (client *HTTPClient) GetWithBody(fullURL, host, cookie string) (*int, *string, error) { - return client.requestWithBody(http.MethodGet, fullURL, host, cookie, "") +func (client *HTTPClient) GetWithBody(fullURL, host, cookie string) (*int, *[]byte, error) { + return client.requestWithBody(http.MethodGet, fullURL, host, cookie, nil) } // PostWithBody gets an URL and returns the status and the body -func (client *HTTPClient) PostWithBody(fullURL, host, cookie, data string) (*int, *string, error) { +func (client *HTTPClient) PostWithBody(fullURL, host, cookie string, data []byte) (*int, *[]byte, error) { return client.requestWithBody(http.MethodPost, fullURL, host, cookie, data) } // requestWithoutBody makes an http request and returns the status, the length and an error -func (client *HTTPClient) requestWithoutBody(method, fullURL, host, cookie, data string) (*int, *int64, error) { +func (client *HTTPClient) requestWithoutBody(method, fullURL, host, cookie string, data []byte) (*int, *int64, error) { resp, err := client.makeRequest(method, fullURL, host, cookie, data) if err != nil { // ignore context canceled errors @@ -137,7 +139,7 @@ func (client *HTTPClient) requestWithoutBody(method, fullURL, host, cookie, data } // requestWithBody makes an http request and returns the status and the body -func (client *HTTPClient) requestWithBody(method, fullURL, host, cookie, data string) (*int, *string, error) { +func (client *HTTPClient) requestWithBody(method, fullURL, host, cookie string, data []byte) (*int, *[]byte, error) { resp, err := client.makeRequest(method, fullURL, host, cookie, data) if err != nil { // ignore context canceled errors @@ -152,12 +154,11 @@ func (client *HTTPClient) requestWithBody(method, fullURL, host, cookie, data st if err != nil { return nil, nil, fmt.Errorf("could not read body: %v", err) } - bodyString := string(body) - return &resp.StatusCode, &bodyString, nil + return &resp.StatusCode, &body, nil } -func (client *HTTPClient) makeRequest(method, fullURL, host, cookie, data string) (*http.Response, error) { +func (client *HTTPClient) makeRequest(method, fullURL, host, cookie string, data []byte) (*http.Response, error) { var req *http.Request var err error @@ -168,7 +169,7 @@ func (client *HTTPClient) makeRequest(method, fullURL, host, cookie, data string return nil, err } case http.MethodPost: - buf := bytes.NewBufferString(data) + buf := bytes.NewBuffer(data) req, err = http.NewRequest(http.MethodPost, fullURL, buf) if err != nil { return nil, err @@ -188,11 +189,11 @@ func (client *HTTPClient) makeRequest(method, fullURL, host, cookie, data string req.Host = host } - ua := DefaultUserAgent() if client.userAgent != "" { - ua = client.userAgent + req.Header.Set("User-Agent", client.userAgent) + } else { + req.Header.Set("User-Agent", defaultUserAgent) } - req.Header.Set("User-Agent", ua) if client.username != "" { req.SetBasicAuth(client.username, client.password) diff --git a/libgobuster/http_test.go b/libgobuster/http_test.go index 9052ec06..957f45fe 100644 --- a/libgobuster/http_test.go +++ b/libgobuster/http_test.go @@ -8,7 +8,15 @@ import ( "testing" ) -func httpServer(t *testing.T, content string) *httptest.Server { +func httpServerB(b *testing.B, content string) *httptest.Server { + b.Helper() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, content) + })) + return ts +} + +func httpServerT(t *testing.T, content string) *httptest.Server { t.Helper() ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, content) @@ -16,8 +24,8 @@ func httpServer(t *testing.T, content string) *httptest.Server { return ts } -func TestMakeRequest(t *testing.T) { - h := httpServer(t, "test") +func TestGet(t *testing.T) { + h := httpServerT(t, "test") defer h.Close() var o HTTPOptions c, err := NewHTTPClient(context.Background(), &o) @@ -35,3 +43,31 @@ func TestMakeRequest(t *testing.T) { t.Fatalf("Invalid length returned: %d", b) } } + +func BenchmarkGet(b *testing.B) { + h := httpServerB(b, "test") + defer h.Close() + var o HTTPOptions + c, err := NewHTTPClient(context.Background(), &o) + if err != nil { + b.Fatalf("Got Error: %v", err) + } + for x := 0; x < b.N; x++ { + _, _, err := c.Get(h.URL, "", "") + if err != nil { + b.Fatalf("Got Error: %v", err) + } + } +} + +func BenchmarkNewHTTPClient(b *testing.B) { + h := httpServerB(b, "test") + defer h.Close() + var o HTTPOptions + for x := 0; x < b.N; x++ { + _, err := NewHTTPClient(context.Background(), &o) + if err != nil { + b.Fatalf("Got Error: %v", err) + } + } +} From 06ee44fb56d5069981b7e2128a698bd9c706786f Mon Sep 17 00:00:00 2001 From: Christian Mehlmauer Date: Wed, 17 Apr 2019 08:57:49 +0200 Subject: [PATCH 3/5] more benchmarks --- helper/helper_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/helper/helper_test.go b/helper/helper_test.go index 447b8719..7f5b5edf 100644 --- a/helper/helper_test.go +++ b/helper/helper_test.go @@ -67,3 +67,50 @@ func TestParseStatusCodes(t *testing.T) { }) } } + +func BenchmarkParseExtensions(b *testing.B) { + var tt = []struct { + testName string + extensions string + expectedExtensions libgobuster.StringSet + expectedError string + }{ + {"Valid extensions", "php,asp,txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Spaces", "php, asp , txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Double extensions", "php,asp,txt,php,asp,txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Leading dot", ".php,asp,.txt", libgobuster.StringSet{Set: map[string]bool{"php": true, "asp": true, "txt": true}}, ""}, + {"Empty string", "", libgobuster.NewStringSet(), "invalid extension string provided"}, + } + + for _, x := range tt { + b.Run(x.testName, func(b2 *testing.B) { + for y := 0; y < b2.N; y++ { + _, _ = ParseExtensions(x.extensions) + } + }) + } +} + +func BenchmarkParseStatusCodes(b *testing.B) { + var tt = []struct { + testName string + stringCodes string + expectedCodes libgobuster.IntSet + expectedError string + }{ + {"Valid codes", "200,100,202", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Spaces", "200, 100 , 202", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Double codes", "200, 100, 202, 100", libgobuster.IntSet{Set: map[int]bool{100: true, 200: true, 202: true}}, ""}, + {"Invalid code", "200,AAA", libgobuster.NewIntSet(), "invalid status code given: AAA"}, + {"Invalid integer", "2000000000000000000000000000000", libgobuster.NewIntSet(), "invalid status code given: 2000000000000000000000000000000"}, + {"Empty string", "", libgobuster.NewIntSet(), "invalid status code string provided"}, + } + + for _, x := range tt { + b.Run(x.testName, func(b2 *testing.B) { + for y := 0; y < b2.N; y++ { + _, _ = ParseStatusCodes(x.stringCodes) + } + }) + } +} From 2c039ef9237fa9e34ee001d8975e659055751684 Mon Sep 17 00:00:00 2001 From: Christian Mehlmauer Date: Wed, 17 Apr 2019 16:26:42 +0200 Subject: [PATCH 4/5] Use Interface instead of byte slice --- libgobuster/http.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libgobuster/http.go b/libgobuster/http.go index 5eef1950..6b3a6add 100644 --- a/libgobuster/http.go +++ b/libgobuster/http.go @@ -1,7 +1,6 @@ package libgobuster import ( - "bytes" "context" "crypto/tls" "fmt" @@ -88,7 +87,7 @@ func (client *HTTPClient) Get(fullURL, host, cookie string) (*int, *int64, error } // Post posts to an URL and returns the status, the length and an error -func (client *HTTPClient) Post(fullURL, host, cookie string, data []byte) (*int, *int64, error) { +func (client *HTTPClient) Post(fullURL, host, cookie string, data io.Reader) (*int, *int64, error) { return client.requestWithoutBody(http.MethodPost, fullURL, host, cookie, data) } @@ -98,12 +97,12 @@ func (client *HTTPClient) GetWithBody(fullURL, host, cookie string) (*int, *[]by } // PostWithBody gets an URL and returns the status and the body -func (client *HTTPClient) PostWithBody(fullURL, host, cookie string, data []byte) (*int, *[]byte, error) { +func (client *HTTPClient) PostWithBody(fullURL, host, cookie string, data io.Reader) (*int, *[]byte, error) { return client.requestWithBody(http.MethodPost, fullURL, host, cookie, data) } // requestWithoutBody makes an http request and returns the status, the length and an error -func (client *HTTPClient) requestWithoutBody(method, fullURL, host, cookie string, data []byte) (*int, *int64, error) { +func (client *HTTPClient) requestWithoutBody(method, fullURL, host, cookie string, data io.Reader) (*int, *int64, error) { resp, err := client.makeRequest(method, fullURL, host, cookie, data) if err != nil { // ignore context canceled errors @@ -139,7 +138,7 @@ func (client *HTTPClient) requestWithoutBody(method, fullURL, host, cookie strin } // requestWithBody makes an http request and returns the status and the body -func (client *HTTPClient) requestWithBody(method, fullURL, host, cookie string, data []byte) (*int, *[]byte, error) { +func (client *HTTPClient) requestWithBody(method, fullURL, host, cookie string, data io.Reader) (*int, *[]byte, error) { resp, err := client.makeRequest(method, fullURL, host, cookie, data) if err != nil { // ignore context canceled errors @@ -158,7 +157,7 @@ func (client *HTTPClient) requestWithBody(method, fullURL, host, cookie string, return &resp.StatusCode, &body, nil } -func (client *HTTPClient) makeRequest(method, fullURL, host, cookie string, data []byte) (*http.Response, error) { +func (client *HTTPClient) makeRequest(method, fullURL, host, cookie string, data io.Reader) (*http.Response, error) { var req *http.Request var err error @@ -169,8 +168,7 @@ func (client *HTTPClient) makeRequest(method, fullURL, host, cookie string, data return nil, err } case http.MethodPost: - buf := bytes.NewBuffer(data) - req, err = http.NewRequest(http.MethodPost, fullURL, buf) + req, err = http.NewRequest(http.MethodPost, fullURL, data) if err != nil { return nil, err } From 4e599f30e66f756ac34dcbf469f861c450283f51 Mon Sep 17 00:00:00 2001 From: Christian Mehlmauer Date: Wed, 17 Apr 2019 21:17:29 +0200 Subject: [PATCH 5/5] overwrite default values, see #127 --- libgobuster/http.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libgobuster/http.go b/libgobuster/http.go index 6b3a6add..b22f6016 100644 --- a/libgobuster/http.go +++ b/libgobuster/http.go @@ -68,7 +68,9 @@ func NewHTTPClient(c context.Context, opt *HTTPOptions) (*HTTPClient, error) { Timeout: opt.Timeout, CheckRedirect: redirectFunc, Transport: &http.Transport{ - Proxy: proxyURLFunc, + Proxy: proxyURLFunc, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, TLSClientConfig: &tls.Config{ InsecureSkipVerify: opt.InsecureSSL, },