Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release/v1.0.3 #21

Merged
merged 8 commits into from
Apr 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file removed .github/workflows/auto-pr.yml
Empty file.
10 changes: 2 additions & 8 deletions .github/workflows/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ on:
workflow_dispatch:
push:
branches:
- dev
- develop
pull_request:
branches:
- main
- dev
- develop
- feature*
paths:
- '**.go'
Expand Down Expand Up @@ -38,12 +38,6 @@ jobs:
run: go test ./...
working-directory: .

#- name: Integration Tests
# env:
# GH_ACTION: true
# run: bash run.sh
# working-directory: integration_tests/

- name: Install
run: go install
working-directory: cmd/hednsextractor/
Expand Down
74 changes: 18 additions & 56 deletions cmd/hednsextractor/hednsextractor.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,7 @@ import (
"strconv"

"github.com/HuntDownProject/hednsextractor/utils"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/levels"
)

var (
Expand All @@ -17,72 +15,35 @@ var (

func main() {

// Parse the stdin
utils.ParseStdin()

flagSet := goflags.NewFlagSet()
flagSet.Marshal = true
flagSet.SetDescription("HEDnsExtractor - A suite for hunting suspicious targets, expose domains and phishing discovery!")
flagSet.BoolVar(&utils.OptionCmd.Onlydomains, "only-domains", false, "show only domains")
flagSet.BoolVar(&utils.OptionCmd.Onlynetworks, "only-networks", false, "show only networks")
flagSet.StringVar(&utils.OptionCmd.Workflow, "workflow", "", "Workflow config")
flagSet.StringVar(&utils.OptionCmd.Target, "target", "", "IP Address or Network to query")
flagSet.BoolVar(&utils.OptionCmd.Silent, "silent", false, "show silent output")
flagSet.BoolVar(&utils.OptionCmd.Verbose, "verbose", false, "show verbose output")

flagSet.CreateGroup("configuration", "Configuration",
flagSet.StringVar(&utils.OptionCmd.Config, "config", utils.DefaultConfigLocation, "flag config file"),
)

flagSet.CreateGroup("config", "Virustotal",
flagSet.BoolVar(&utils.OptionCmd.Vtscore, "vt", false, "show Virustotal score"),
flagSet.StringVar(&utils.OptionCmd.VtApiKey, "vt-api-key", "", "Virustotal API Key"),
flagSet.StringVar(&utils.OptionCmd.VtscoreValue, "vt-score", "0", "Minimum Virustotal score to show"),
)

if err := flagSet.Parse(); err != nil {
gologger.Fatal().Msgf("Could not parse flags: %s\n", err)
}

if utils.OptionCmd.Verbose {
gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose)
} else if utils.OptionCmd.Silent {
gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent)
} else {
gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo)
}

if utils.OptionCmd.Vtscore && utils.OptionCmd.VtApiKey == "" {
gologger.Fatal().Msgf("A Virustotal API Key is needed in config file: %s\n", utils.DefaultConfigLocation)
}
// Load parameters from command line and configuration file
utils.LoadParameters()

// Show Banner
utils.ShowBanner()

// read the targets from yaml
var c utils.Conf
// read the Workflow from yaml
var workflow utils.Workflow
if utils.OptionCmd.Workflow != "" {
c.GetConf(utils.OptionCmd.Workflow)
workflow.GetConf(utils.OptionCmd.Workflow)

for i := range c.Domains {
utils.IdentifyTarget(c.Domains[i])
for i := range workflow.Domains {
utils.IdentifyTarget(workflow.Domains[i])
}

for i := range c.Ipaddrs {
utils.IdentifyTarget(c.Ipaddrs[i])
for i := range workflow.Ipaddrs {
utils.IdentifyTarget(workflow.Ipaddrs[i])
}

for i := range c.Networks {
utils.IdentifyTarget(c.Networks[i])
for i := range workflow.Networks {
utils.IdentifyTarget(workflow.Networks[i])
}
}

// Look into target parameter to grab the IPv4s and Networks
if utils.OptionCmd.Target != "" {
gologger.Verbose().Msgf("Identifying networks for %s", utils.OptionCmd.Target)
utils.IdentifyTarget(utils.OptionCmd.Target)
}

utils.RunCrawler()
hurricane := utils.Hurricane{}
hurricane.RunCrawler()

if utils.OptionCmd.Vtscore && !utils.OptionCmd.Silent {
gologger.Info().Msgf("Filtering with Virustotal with a mininum score %s", utils.OptionCmd.VtscoreValue)
Expand All @@ -92,8 +53,8 @@ func main() {
var bMatchedPTR = false
var bMatchedDomain = false

if c.Regex != "" {
var re = regexp.MustCompile(c.Regex)
if workflow.Regex != "" {
var re = regexp.MustCompile(workflow.Regex)
bMatchedDomain = re.MatchString(result.Domain)
bMatchedPTR = re.MatchString(result.PTR)
} else {
Expand All @@ -106,7 +67,8 @@ func main() {
}

if utils.OptionCmd.Vtscore {
result.VtScore = utils.GetVtReport(result.Domain)
virustotal := utils.Virustotal{}
result.VtScore = virustotal.GetVtReport(result.Domain)
if score, err := strconv.ParseUint(utils.OptionCmd.VtscoreValue, 10, 64); err == nil {
if result.VtScore < score {
continue
Expand Down
61 changes: 36 additions & 25 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,57 @@ module github.com/HuntDownProject/hednsextractor
go 1.21

require (
github.com/PuerkitoBio/goquery v1.8.1
github.com/projectdiscovery/goflags v0.1.32
github.com/PuerkitoBio/goquery v1.9.1
github.com/projectdiscovery/goflags v0.1.46
github.com/projectdiscovery/gologger v1.1.12
github.com/projectdiscovery/retryablehttp-go v1.0.22
github.com/projectdiscovery/utils v0.0.68
github.com/tidwall/gjson v1.17.0
github.com/projectdiscovery/retryablehttp-go v1.0.54
github.com/projectdiscovery/utils v0.0.87
github.com/tidwall/gjson v1.17.1
gopkg.in/yaml.v2 v2.4.0
)

require (
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
github.com/akrylysov/pogreb v0.10.2 // indirect
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/chzyer/readline v1.5.1 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
github.com/gaukas/godicttls v0.0.3 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.4 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
github.com/mholt/archiver/v3 v3.5.1 // indirect
github.com/microcosm-cc/bluemonday v1.0.26 // indirect
github.com/miekg/dns v1.1.57 // indirect
github.com/miekg/dns v1.1.58 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nwaples/rardecode v1.1.3 // indirect
github.com/onsi/gomega v1.22.1 // indirect
github.com/pierrec/lz4/v4 v4.1.19 // indirect
github.com/onsi/ginkgo/v2 v2.17.1 // indirect
github.com/onsi/gomega v1.32.0 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
github.com/projectdiscovery/fastdialer v0.0.35 // indirect
github.com/projectdiscovery/hmap v0.0.30 // indirect
github.com/projectdiscovery/networkpolicy v0.0.6 // indirect
github.com/projectdiscovery/retryabledns v1.0.47 // indirect
github.com/refraction-networking/utls v1.3.3 // indirect
github.com/projectdiscovery/fastdialer v0.0.65 // indirect
github.com/projectdiscovery/hmap v0.0.41 // indirect
github.com/projectdiscovery/networkpolicy v0.0.8 // indirect
github.com/projectdiscovery/retryabledns v1.0.58 // indirect
github.com/quic-go/quic-go v0.42.0 // indirect
github.com/refraction-networking/utls v1.6.3 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/tidwall/btree v1.7.0 // indirect
Expand All @@ -53,22 +63,23 @@ require (
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/rtred v0.1.2 // indirect
github.com/tidwall/tinyqueue v0.1.1 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 // indirect
github.com/weppos/publicsuffix-go v0.30.2 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/yl2chen/cidranger v1.0.2 // indirect
github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect
go.etcd.io/bbolt v1.3.8 // indirect
go.etcd.io/bbolt v1.3.9 // indirect
go.uber.org/mock v0.4.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect
golang.org/x/mod v0.16.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.16.1 // indirect
golang.org/x/tools v0.19.0 // indirect
gopkg.in/djherbis/times.v1 v1.3.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading
Loading