From 62085874972ceb48a7dcc345a36add84f569a6c1 Mon Sep 17 00:00:00 2001 From: Neriberto CP Date: Wed, 20 Mar 2024 21:16:08 -0300 Subject: [PATCH] Refactoring the code Refactoring the code --- cmd/hednsextractor/hednsextractor.go | 74 ++++---------- utils/data.go | 26 ----- utils/hurricane.go | 137 +++++++++++++++++++++++++ utils/network.go | 148 --------------------------- utils/parameters.go | 67 ++++++++++++ utils/virustotal.go | 27 +++-- utils/workflow.go | 29 ++++++ 7 files changed, 270 insertions(+), 238 deletions(-) create mode 100644 utils/hurricane.go create mode 100644 utils/parameters.go create mode 100644 utils/workflow.go diff --git a/cmd/hednsextractor/hednsextractor.go b/cmd/hednsextractor/hednsextractor.go index 249bdbb..d6f6d4d 100644 --- a/cmd/hednsextractor/hednsextractor.go +++ b/cmd/hednsextractor/hednsextractor.go @@ -6,9 +6,7 @@ import ( "strconv" "github.com/HuntDownProject/hednsextractor/utils" - "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" ) var ( @@ -17,72 +15,35 @@ var ( func main() { + // Parse the stdin utils.ParseStdin() - flagSet := goflags.NewFlagSet() - flagSet.Marshal = true - flagSet.SetDescription("HEDnsExtractor - A suite for hunting suspicious targets, expose domains and phishing discovery!") - flagSet.BoolVar(&utils.OptionCmd.Onlydomains, "only-domains", false, "show only domains") - flagSet.BoolVar(&utils.OptionCmd.Onlynetworks, "only-networks", false, "show only networks") - flagSet.StringVar(&utils.OptionCmd.Workflow, "workflow", "", "Workflow config") - flagSet.StringVar(&utils.OptionCmd.Target, "target", "", "IP Address or Network to query") - flagSet.BoolVar(&utils.OptionCmd.Silent, "silent", false, "show silent output") - flagSet.BoolVar(&utils.OptionCmd.Verbose, "verbose", false, "show verbose output") - - flagSet.CreateGroup("configuration", "Configuration", - flagSet.StringVar(&utils.OptionCmd.Config, "config", utils.DefaultConfigLocation, "flag config file"), - ) - - flagSet.CreateGroup("config", "Virustotal", - flagSet.BoolVar(&utils.OptionCmd.Vtscore, "vt", false, "show Virustotal score"), - flagSet.StringVar(&utils.OptionCmd.VtApiKey, "vt-api-key", "", "Virustotal API Key"), - flagSet.StringVar(&utils.OptionCmd.VtscoreValue, "vt-score", "0", "Minimum Virustotal score to show"), - ) - - if err := flagSet.Parse(); err != nil { - gologger.Fatal().Msgf("Could not parse flags: %s\n", err) - } - - if utils.OptionCmd.Verbose { - gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) - } else if utils.OptionCmd.Silent { - gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) - } else { - gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) - } - - if utils.OptionCmd.Vtscore && utils.OptionCmd.VtApiKey == "" { - gologger.Fatal().Msgf("A Virustotal API Key is needed in config file: %s\n", utils.DefaultConfigLocation) - } + // Load parameters from command line and configuration file + utils.LoadParameters() // Show Banner utils.ShowBanner() - // read the targets from yaml - var c utils.Conf + // read the Workflow from yaml + var workflow utils.Workflow if utils.OptionCmd.Workflow != "" { - c.GetConf(utils.OptionCmd.Workflow) + workflow.GetConf(utils.OptionCmd.Workflow) - for i := range c.Domains { - utils.IdentifyTarget(c.Domains[i]) + for i := range workflow.Domains { + utils.IdentifyTarget(workflow.Domains[i]) } - for i := range c.Ipaddrs { - utils.IdentifyTarget(c.Ipaddrs[i]) + for i := range workflow.Ipaddrs { + utils.IdentifyTarget(workflow.Ipaddrs[i]) } - for i := range c.Networks { - utils.IdentifyTarget(c.Networks[i]) + for i := range workflow.Networks { + utils.IdentifyTarget(workflow.Networks[i]) } } - // Look into target parameter to grab the IPv4s and Networks - if utils.OptionCmd.Target != "" { - gologger.Verbose().Msgf("Identifying networks for %s", utils.OptionCmd.Target) - utils.IdentifyTarget(utils.OptionCmd.Target) - } - - utils.RunCrawler() + hurricane := utils.Hurricane{} + hurricane.RunCrawler() if utils.OptionCmd.Vtscore && !utils.OptionCmd.Silent { gologger.Info().Msgf("Filtering with Virustotal with a mininum score %s", utils.OptionCmd.VtscoreValue) @@ -92,8 +53,8 @@ func main() { var bMatchedPTR = false var bMatchedDomain = false - if c.Regex != "" { - var re = regexp.MustCompile(c.Regex) + if workflow.Regex != "" { + var re = regexp.MustCompile(workflow.Regex) bMatchedDomain = re.MatchString(result.Domain) bMatchedPTR = re.MatchString(result.PTR) } else { @@ -106,7 +67,8 @@ func main() { } if utils.OptionCmd.Vtscore { - result.VtScore = utils.GetVtReport(result.Domain) + virustotal := utils.Virustotal{} + result.VtScore = virustotal.GetVtReport(result.Domain) if score, err := strconv.ParseUint(utils.OptionCmd.VtscoreValue, 10, 64); err == nil { if result.VtScore < score { continue diff --git a/utils/data.go b/utils/data.go index 9f1021e..5e63e88 100644 --- a/utils/data.go +++ b/utils/data.go @@ -1,24 +1,11 @@ package utils import ( - "log" - "os" "path/filepath" - "github.com/projectdiscovery/gologger" folderutil "github.com/projectdiscovery/utils/folder" - "gopkg.in/yaml.v2" ) -const urlBase = "https://bgp.he.net/" - -type Conf struct { - Domains []string `yaml:"domains"` - Ipaddrs []string `yaml:"ipaddrs"` - Networks []string `yaml:"networks"` - Regex string `yaml:"regex"` -} - type Result struct { IPAddr string PTR string @@ -36,16 +23,3 @@ var ( ) var Results = make(map[string]Result) - -func (c *Conf) GetConf(filename string) *Conf { - yamlFile, err := os.ReadFile(filename) - if err != nil { - gologger.Fatal().Msgf("Could not %s\n", err) - } - - err = yaml.Unmarshal(yamlFile, c) - if err != nil { - log.Fatalf("Unmarshal: %v", err) - } - return c -} diff --git a/utils/hurricane.go b/utils/hurricane.go new file mode 100644 index 0000000..581e6ec --- /dev/null +++ b/utils/hurricane.go @@ -0,0 +1,137 @@ +package utils + +import ( + "io" + "net/http/httputil" + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/retryablehttp-go" +) + +type Hurricane struct { +} + +const urlBase = "https://bgp.he.net/" + +func (h *Hurricane) RunCrawler() { + for _, domain := range Domains { + gologger.Verbose().Msgf("Identifying networks for domain: %s", domain) + h.ExtractDomain(domain) + } + + for _, host := range Hosts { + gologger.Verbose().Msgf("Identifying networks for IPv4: %s", host) + h.ExtractNetwork(host) + } + + if !OptionCmd.Onlynetworks { + for _, network := range Networks { + gologger.Verbose().Msgf("Identifying domains for network: %s", network) + h.ExtractDomains(network) + } + } +} + +func (h *Hurricane) Request(url string) string { + opts := retryablehttp.DefaultOptionsSpraying + client := retryablehttp.NewClient(opts) + resp, err := client.Get(url) + if err != nil { + panic(err) + } + + bin, err := httputil.DumpResponse(resp, true) + if err != nil { + panic(err) + } + str := string(bin) + + return str +} + +func (h *Hurricane) ExtractDomain(domain string) { + var url = "" + + if domain != "" { + url = urlBase + "dns/" + domain + } + + var str = h.Request(url) + + var re = regexp.MustCompile(`(?m)href="/net/([^"]+)"`) + for _, match := range re.FindAllStringSubmatch(str, -1) { + if !Contains(Networks, match[1]) { + if (!OptionCmd.Silent && !OptionCmd.Onlydomains) || OptionCmd.Onlynetworks { + gologger.Info().Msgf("[%s] network: %s\n", domain, match[1]) + } + Networks = append(Networks, match[1]) + } + } +} + +func (h *Hurricane) ExtractDomains(ipRange string) { + if ipRange == "" { + return + } + + var url = urlBase + "net/" + ipRange + var html = h.Request(url) + + h.ParseHTML(strings.NewReader(html)) +} + +func (h *Hurricane) ParseHTML(body io.Reader) { + doc, err := goquery.NewDocumentFromReader(body) + if err != nil { + gologger.Fatal().Msgf("%s", err) + } + var re = regexp.MustCompile(`\/dns\/([^"]+)`) + doc.Find("#dnsrecords").Each(func(h int, div *goquery.Selection) { + div.Find("tr").Each(func(i int, tr *goquery.Selection) { + var result Result + tr.Find("td").Each(func(j int, td *goquery.Selection) { + td.Find("a").Each(func(k int, a *goquery.Selection) { + switch td.Index() { + case 0: + result.IPAddr = a.Text() + case 1: + result.PTR = a.Text() + case 2: + html, err := td.Html() + if err == nil { + for _, match := range re.FindAllStringSubmatch(html, -1) { + result.Domain = match[1] + Results[result.Domain] = result + } + } + } + }) + }) + }) + }) +} + +func (h *Hurricane) ExtractNetwork(ip string) { + var url = "" + + if ip != "" { + url = urlBase + "ip/" + ip + } + + var str = h.Request(url) + + if ip != "" { + var re = regexp.MustCompile(`(?m)href="/net/([^"]+)"`) + for _, match := range re.FindAllStringSubmatch(str, -1) { + if !Contains(Networks, match[1]) { + if (!OptionCmd.Silent && !OptionCmd.Onlydomains) || OptionCmd.Onlynetworks { + gologger.Info().Msgf("[%s] network: %s\n", ip, match[1]) + } + Networks = append(Networks, match[1]) + } + } + } +} diff --git a/utils/network.go b/utils/network.go index d606ca6..c6c18ab 100644 --- a/utils/network.go +++ b/utils/network.go @@ -1,19 +1,7 @@ package utils import ( - "bufio" - "io" "net" - "net/http/httputil" - "os" - "regexp" - "strings" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/retryablehttp-go" - fileutil "github.com/projectdiscovery/utils/file" - - "github.com/PuerkitoBio/goquery" ) func Contains(s []string, e string) bool { @@ -25,16 +13,6 @@ func Contains(s []string, e string) bool { return false } -func ParseStdin() { - // Look into stdin to grab the IPv4s and Networks - if fileutil.HasStdin() { - s := bufio.NewScanner(os.Stdin) - for s.Scan() { - IdentifyTarget(s.Text()) - } - } -} - func IsIpAddr(ipAddr string) bool { addr := net.ParseIP(ipAddr) return addr != nil @@ -57,129 +35,3 @@ func IdentifyTarget(target string) { Domains = append(Domains, target) } } - -func Request(url string) string { - opts := retryablehttp.DefaultOptionsSpraying - client := retryablehttp.NewClient(opts) - resp, err := client.Get(url) - if err != nil { - panic(err) - } - - bin, err := httputil.DumpResponse(resp, true) - if err != nil { - panic(err) - } - str := string(bin) - - return str -} - -func ParseHTML(body io.Reader) { - doc, err := goquery.NewDocumentFromReader(body) - if err != nil { - gologger.Fatal().Msgf("%s", err) - } - - var re = regexp.MustCompile(`\/dns\/([^"]+)`) - - doc.Find("#dnsrecords").Each(func(h int, div *goquery.Selection) { - div.Find("tr").Each(func(i int, tr *goquery.Selection) { - var result Result - tr.Find("td").Each(func(j int, td *goquery.Selection) { - td.Find("a").Each(func(k int, a *goquery.Selection) { - switch td.Index() { - case 0: - result.IPAddr = a.Text() - case 1: - result.PTR = a.Text() - case 2: - html, err := td.Html() - if err == nil { - for _, match := range re.FindAllStringSubmatch(html, -1) { - result.Domain = match[1] - Results[result.Domain] = result - } - } - } - }) - }) - }) - }) -} - -func ExtractDomain(domain string, silent bool) { - var url = "" - - if domain != "" { - url = urlBase + "dns/" + domain - } - - var str = Request(url) - - var re = regexp.MustCompile(`(?m)href="/net/([^"]+)"`) - for _, match := range re.FindAllStringSubmatch(str, -1) { - if !Contains(Networks, match[1]) { - if (!silent && !OptionCmd.Onlydomains) || OptionCmd.Onlynetworks { - gologger.Info().Msgf("[%s] network: %s\n", domain, match[1]) - } - Networks = append(Networks, match[1]) - } - } -} - -func ExtractNetwork(ip string, silent bool, onlydomains bool, onlynetworks bool) { - var url = "" - - if ip != "" { - url = urlBase + "ip/" + ip - } - - var str = Request(url) - - if ip != "" { - var re = regexp.MustCompile(`(?m)href="/net/([^"]+)"`) - for _, match := range re.FindAllStringSubmatch(str, -1) { - if !Contains(Networks, match[1]) { - if (!silent && !onlydomains) || onlynetworks { - gologger.Info().Msgf("[%s] network: %s\n", ip, match[1]) - } - Networks = append(Networks, match[1]) - } - } - } -} - -func ExtractDomains(ipRange string) { - if ipRange == "" { - return - } - - var url = urlBase + "net/" + ipRange - var html = Request(url) - - ParseHTML(strings.NewReader(html)) -} - -func RunCrawler() { - for _, domain := range Domains { - gologger.Verbose().Msgf("Identifying networks for domain: %s", domain) - ExtractDomain(domain, OptionCmd.Silent) - } - - for _, host := range Hosts { - gologger.Verbose().Msgf("Identifying networks for IPv4: %s", host) - ExtractNetwork( - host, - OptionCmd.Silent, - OptionCmd.Onlydomains, - OptionCmd.Onlynetworks) - } - - if !OptionCmd.Onlynetworks { - for _, network := range Networks { - gologger.Verbose().Msgf("Identifying domains for network: %s", network) - ExtractDomains(network) - } - } -} diff --git a/utils/parameters.go b/utils/parameters.go new file mode 100644 index 0000000..5f66bf9 --- /dev/null +++ b/utils/parameters.go @@ -0,0 +1,67 @@ +package utils + +import ( + "bufio" + "os" + + fileutil "github.com/projectdiscovery/utils/file" + + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" +) + +func ParseStdin() { + // Look into stdin to grab the IPv4s and Networks + if fileutil.HasStdin() { + s := bufio.NewScanner(os.Stdin) + for s.Scan() { + IdentifyTarget(s.Text()) + } + } +} + +func LoadParameters() { + // Load parameters from a file + flagSet := goflags.NewFlagSet() + flagSet.Marshal = true + flagSet.SetDescription("HEDnsExtractor - A suite for hunting suspicious targets, expose domains and phishing discovery!") + flagSet.BoolVar(&OptionCmd.Onlydomains, "only-domains", false, "show only domains") + flagSet.BoolVar(&OptionCmd.Onlynetworks, "only-networks", false, "show only networks") + flagSet.StringVar(&OptionCmd.Workflow, "workflow", "", "Workflow config") + flagSet.StringVar(&OptionCmd.Target, "target", "", "IP Address or Network to query") + flagSet.BoolVar(&OptionCmd.Silent, "silent", false, "show silent output") + flagSet.BoolVar(&OptionCmd.Verbose, "verbose", false, "show verbose output") + + flagSet.CreateGroup("configuration", "Configuration", + flagSet.StringVar(&OptionCmd.Config, "config", DefaultConfigLocation, "flag config file"), + ) + + flagSet.CreateGroup("config", "Virustotal", + flagSet.BoolVar(&OptionCmd.Vtscore, "vt", false, "show Virustotal score"), + flagSet.StringVar(&OptionCmd.VtApiKey, "vt-api-key", "", "Virustotal API Key"), + flagSet.StringVar(&OptionCmd.VtscoreValue, "vt-score", "0", "Minimum Virustotal score to show"), + ) + + if err := flagSet.Parse(); err != nil { + gologger.Fatal().Msgf("Could not parse flags: %s\n", err) + } + + if OptionCmd.Verbose { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) + } else if OptionCmd.Silent { + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + } else { + gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) + } + + if OptionCmd.Vtscore && OptionCmd.VtApiKey == "" { + gologger.Fatal().Msgf("A Virustotal API Key is needed in config file: %s\n", DefaultConfigLocation) + } + + // Look into target parameter to grab the IPv4s and Networks + if OptionCmd.Target != "" { + gologger.Verbose().Msgf("Identifying networks for %s", OptionCmd.Target) + IdentifyTarget(OptionCmd.Target) + } +} diff --git a/utils/virustotal.go b/utils/virustotal.go index dd41d22..ff9850a 100644 --- a/utils/virustotal.go +++ b/utils/virustotal.go @@ -2,15 +2,30 @@ package utils import ( "fmt" + "net/http/httputil" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/retryablehttp-go" "github.com/tidwall/gjson" - "net/http/httputil" ) -func GetVtReport(domain string) uint64 { - urlBase := fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s", domain) +type Virustotal struct { +} + +const vt_url = "https://www.virustotal.com" + +func (c *Virustotal) GetVtReport(domain string) uint64 { + urlBase := fmt.Sprintf("%s/api/v3/domains/%s", vt_url, domain) + + str := c.request(urlBase) + + malicious := gjson.Get(str, "data.attributes.last_analysis_stats.malicious") + suspicious := gjson.Get(str, "data.attributes.last_analysis_stats.suspicious") + vtScore := malicious.Uint() + suspicious.Uint() + return vtScore +} +func (c *Virustotal) request(urlBase string) string { request, err := retryablehttp.NewRequest("GET", urlBase, nil) if err != nil { gologger.Fatal().Msgf("err: %v", err) @@ -29,9 +44,5 @@ func GetVtReport(domain string) uint64 { panic(err) } str := string(bin) - - malicious := gjson.Get(str, "data.attributes.last_analysis_stats.malicious") - suspicious := gjson.Get(str, "data.attributes.last_analysis_stats.suspicious") - vtScore := malicious.Uint() + suspicious.Uint() - return vtScore + return str } diff --git a/utils/workflow.go b/utils/workflow.go new file mode 100644 index 0000000..ed789df --- /dev/null +++ b/utils/workflow.go @@ -0,0 +1,29 @@ +package utils + +import ( + "log" + "os" + + "github.com/projectdiscovery/gologger" + "gopkg.in/yaml.v2" +) + +type Workflow struct { + Domains []string `yaml:"domains"` + Ipaddrs []string `yaml:"ipaddrs"` + Networks []string `yaml:"networks"` + Regex string `yaml:"regex"` +} + +func (c *Workflow) GetConf(filename string) *Workflow { + yamlFile, err := os.ReadFile(filename) + if err != nil { + gologger.Fatal().Msgf("Could not %s\n", err) + } + + err = yaml.Unmarshal(yamlFile, c) + if err != nil { + log.Fatalf("Unmarshal: %v", err) + } + return c +}