From 9b5fef380f12deddac1f2aad26f314e0b90ca1d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 10:08:42 +0000 Subject: [PATCH 001/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.11 to 0.0.12. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.11...v0.0.12) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a550e3f..3cf317e 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.11 + github.com/projectdiscovery/wappalyzergo v0.0.12 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 8e022c3..70c0617 100644 --- a/go.sum +++ b/go.sum @@ -189,8 +189,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.11 h1:KKHZq5PKk2Xq23AuybveeizFz2jYrRbsDscNxkcuKCg= -github.com/projectdiscovery/wappalyzergo v0.0.11/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.12 h1:/xSsQBa1Ibt72bsHtUP6zHRZL1yyflj+rANdJ4QuRdI= +github.com/projectdiscovery/wappalyzergo v0.0.12/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= From 0bebb4aec34dbbe807a30ea97b04eff0eafbc316 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 30 Aug 2021 17:13:22 +0200 Subject: [PATCH 002/355] Disabling automatic UA if user defines one --- common/customheader/customheader.go | 15 +++++++++++++++ go.mod | 2 +- go.sum | 2 ++ runner/runner.go | 6 +++++- 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/common/customheader/customheader.go b/common/customheader/customheader.go index 133b6fe..8d17652 100644 --- a/common/customheader/customheader.go +++ b/common/customheader/customheader.go @@ -1,5 +1,9 @@ package customheader +import ( + "github.com/projectdiscovery/stringsutil" +) + // CustomHeaders valid for all requests type CustomHeaders []string @@ -13,3 +17,14 @@ func (c *CustomHeaders) Set(value string) error { *c = append(*c, value) return nil } + +// Has checks if the list contains a header name +func (c *CustomHeaders) Has(header string) bool { + for _, customHeader := range *c { + if stringsutil.HasPrefixAny(customHeader, header) { + return true + } + } + + return false +} diff --git a/go.mod b/go.mod index 3cf317e..597043d 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/retryabledns v1.0.12 // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea - github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe + github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 github.com/projectdiscovery/wappalyzergo v0.0.12 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 70c0617..953840a 100644 --- a/go.sum +++ b/go.sum @@ -187,6 +187,8 @@ github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= +github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= github.com/projectdiscovery/wappalyzergo v0.0.12 h1:/xSsQBa1Ibt72bsHtUP6zHRZL1yyflj+rANdJ4QuRdI= diff --git a/runner/runner.go b/runner/runner.go index 55caab2..ae10f42 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -91,7 +91,11 @@ func New(options *Options) (*Runner, error) { httpxOptions.UnsafeURI = options.RequestURI httpxOptions.CdnCheck = options.OutputCDN httpxOptions.ExcludeCdn = options.ExcludeCDN - httpxOptions.RandomAgent = options.RandomAgent + if options.CustomHeaders.Has("User-Agent:") { + httpxOptions.RandomAgent = false + } else { + httpxOptions.RandomAgent = options.RandomAgent + } httpxOptions.Deny = options.Deny httpxOptions.Allow = options.Allow httpxOptions.MaxResponseBodySizeToSave = int64(options.MaxResponseBodySizeToSave) From fc77c543951aea9960918841715c26396cdf905a Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 3 Sep 2021 15:50:17 +0200 Subject: [PATCH 003/355] Fixing content-length calculation --- common/httpx/httpx.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index ac0986f..ca255fd 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -194,13 +194,18 @@ get_response: respbodystr = h.htmlPolicy.Sanitize(respbodystr) } - if contentLength, ok := resp.Headers["Content-Length"]; ok { - contentLengthInt, err := strconv.Atoi(strings.Join(contentLength, "")) - if err != nil { - resp.ContentLength = utf8.RuneCountInString(respbodystr) - } else { + // if content length is not defined + if resp.ContentLength <= 0 { + // check if it's in the header and convert to int + if contentLength, ok := resp.Headers["Content-Length"]; ok { + contentLengthInt, _ := strconv.Atoi(strings.Join(contentLength, "")) resp.ContentLength = contentLengthInt } + + // if we have a body, then use the number of bytes in the body if the length is still zero + if resp.ContentLength <= 0 && len(respbodystr) > 0 { + resp.ContentLength = utf8.RuneCountInString(respbodystr) + } } resp.Data = respbody From 3788cf0478bb85fa9696368b03a97187df900e88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 10:08:16 +0000 Subject: [PATCH 004/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.12 to 0.0.13. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.12...v0.0.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3cf317e..11f1cd8 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.12 + github.com/projectdiscovery/wappalyzergo v0.0.13 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 70c0617..9f9e9c3 100644 --- a/go.sum +++ b/go.sum @@ -189,8 +189,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.12 h1:/xSsQBa1Ibt72bsHtUP6zHRZL1yyflj+rANdJ4QuRdI= -github.com/projectdiscovery/wappalyzergo v0.0.12/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.13 h1:00Kc1QE5YGqd8u6jd3E8SZQk3eQhjGyme5g2eAJtdCY= +github.com/projectdiscovery/wappalyzergo v0.0.13/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= From ca6eed1f4729562f8046da6766f3526068762ca5 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Thu, 9 Sep 2021 00:41:24 -0400 Subject: [PATCH 005/355] build: bump to go 1.17 --- .github/workflows/build-test.yml | 15 +++++++------- .github/workflows/release-binary.yml | 30 ++++++++++++---------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 88045ab..3f76a5b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,22 +1,21 @@ name: 🔨 Build Test + on: push: pull_request: workflow_dispatch: -jobs: +jobs: build: name: Test Builds runs-on: ubuntu-latest steps: - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.15 + - uses: actions/checkout@v2 - - name: Check out code - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: 1.17 - name: Test run: go test ./... @@ -27,4 +26,4 @@ jobs: - name: Build run: go build . - working-directory: cmd/httpx/ \ No newline at end of file + working-directory: cmd/httpx/ diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 6fe8c82..7cf8f99 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -5,26 +5,22 @@ on: - v* workflow_dispatch: -jobs: - release: +jobs: + release: runs-on: ubuntu-latest - steps: - - - name: "Check out code" - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: "Set up Go" - uses: actions/setup-go@v2 - with: - go-version: 1.16 - - - env: + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - + env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v2 - with: + with: args: "release --rm-dist" version: latest - workdir: . \ No newline at end of file + workdir: . From d328ab7d560cf2348498412c6497633df72efaf3 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Thu, 9 Sep 2021 00:42:14 -0400 Subject: [PATCH 006/355] build: update go.mod and pull in indirect dependencies --- go.mod | 24 +++++++++++++++++++++++- go.sum | 3 --- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a550e3f..9074db8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/httpx -go 1.14 +go 1.17 require ( github.com/akrylysov/pogreb v0.10.1 // indirect @@ -45,3 +45,25 @@ require ( golang.org/x/text v0.3.7 google.golang.org/protobuf v1.27.1 // indirect ) + +require ( + github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/dgraph-io/badger v1.6.2 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/karrick/godirwalk v1.16.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect + github.com/projectdiscovery/networkpolicy v0.0.1 // indirect + github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect + github.com/syndtr/goleveldb v1.0.0 // indirect + github.com/yl2chen/cidranger v1.0.2 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect +) diff --git a/go.sum b/go.sum index 8e022c3..1b570e5 100644 --- a/go.sum +++ b/go.sum @@ -92,11 +92,9 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= @@ -312,7 +310,6 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= From 3ca698a571bb6e3a84bf8ac4f3254699e3937141 Mon Sep 17 00:00:00 2001 From: Rui Chen Date: Thu, 9 Sep 2021 00:43:16 -0400 Subject: [PATCH 007/355] docker: use tagged image rather than latest Signed-off-by: Rui Chen --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2928e1c..1886ef6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,7 @@ FROM golang:1.17.0-alpine AS builder RUN apk add --no-cache git RUN GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx -FROM alpine:latest - +FROM alpine:3.14 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From 5c10ddf8c229a1d29476e939267d5819ddd7fedf Mon Sep 17 00:00:00 2001 From: wux1an Date: Thu, 9 Sep 2021 18:55:31 +0800 Subject: [PATCH 008/355] feat: Support output in CSV format --- runner/options.go | 7 +++ runner/runner.go | 126 +++++++++++++++++++++++++++++++++------------- 2 files changed, 98 insertions(+), 35 deletions(-) diff --git a/runner/options.go b/runner/options.go index f0678b5..e9a73a9 100644 --- a/runner/options.go +++ b/runner/options.go @@ -148,6 +148,7 @@ type Options struct { FollowRedirects bool StoreResponse bool JSONOutput bool + CSVOutput bool Silent bool Version bool Verbose bool @@ -217,6 +218,7 @@ func ParseOptions() *Options { flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host") flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") flag.BoolVar(&options.JSONOutput, "json", false, "Display output in JSON format") + flag.BoolVar(&options.CSVOutput, "csv", false, "Display output in CSV format") flag.StringVar(&options.InputFile, "l", "", "Input file containing list of hosts to process") flag.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods") flag.BoolVar(&options.OutputMethod, "method", false, "Display request method") @@ -299,6 +301,11 @@ func (options *Options) validateOptions() { gologger.Fatal().Msgf("File %s does not exist!\n", options.InputRawRequest) } + multiOutput := options.CSVOutput && options.JSONOutput + if multiOutput { + gologger.Fatal().Msg("Results can only be displayed in one format: 'JSON' or 'CSV'\n") + } + var err error if options.matchStatusCode, err = stringz.StringToSliceInt(options.OutputMatchStatusCode); err != nil { gologger.Fatal().Msgf("Invalid value for match status code option: %s\n", err) diff --git a/runner/runner.go b/runner/runner.go index 55caab2..2980271 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "crypto/sha256" + "encoding/csv" "encoding/hex" "encoding/json" "fmt" @@ -14,6 +15,7 @@ import ( "net/url" "os" "path" + "reflect" "regexp" "sort" "strconv" @@ -432,6 +434,15 @@ func (r *Runner) RunEnumeration() { } defer f.Close() //nolint } + if r.options.CSVOutput { + header := Result{}.CSVHeader() + gologger.Silent().Msgf("%s\n", header) + if f != nil { + //nolint:errcheck // this method needs a small refactor to reduce complexity + f.WriteString(header + "\n") + } + } + for resp := range output { if resp.err != nil { gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.err) @@ -469,7 +480,10 @@ func (r *Runner) RunEnumeration() { row := resp.str if r.options.JSONOutput { row = resp.JSON(&r.scanopts) + } else if r.options.CSVOutput { + row = resp.CSVRow(&r.scanopts) } + gologger.Silent().Msgf("%s\n", row) if f != nil { //nolint:errcheck // this method needs a small refactor to reduce complexity @@ -1112,44 +1126,44 @@ func (r *Runner) SaveResumeConfig() error { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty"` - Request string `json:"request,omitempty"` - ResponseHeader string `json:"response-header,omitempty"` - Scheme string `json:"scheme,omitempty"` - Port string `json:"port,omitempty"` - Path string `json:"path,omitempty"` - BodySHA256 string `json:"body-sha256,omitempty"` - HeaderSHA256 string `json:"header-sha256,omitempty"` - A []string `json:"a,omitempty"` - CNAMEs []string `json:"cnames,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` + Request string `json:"request,omitempty" csv:"request"` + ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` + Scheme string `json:"scheme,omitempty" csv:"scheme"` + Port string `json:"port,omitempty" csv:"port"` + Path string `json:"path,omitempty" csv:"path"` + BodySHA256 string `json:"body-sha256,omitempty" csv:"body-sha256"` + HeaderSHA256 string `json:"header-sha256,omitempty" csv:"header-sha256"` + A []string `json:"a,omitempty" csv:"a"` + CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` raw string - URL string `json:"url,omitempty"` - Input string `json:"input,omitempty"` - Location string `json:"location,omitempty"` - Title string `json:"title,omitempty"` + URL string `json:"url,omitempty" csv:"url"` + Input string `json:"input,omitempty" csv:"input"` + Location string `json:"location,omitempty" csv:"location"` + Title string `json:"title,omitempty" csv:"title"` str string err error - Error string `json:"error,omitempty"` - WebServer string `json:"webserver,omitempty"` - ResponseBody string `json:"response-body,omitempty"` - ContentType string `json:"content-type,omitempty"` - Method string `json:"method,omitempty"` - Host string `json:"host,omitempty"` - ContentLength int `json:"content-length,omitempty"` - ChainStatusCodes []int `json:"chain-status-codes,omitempty"` - StatusCode int `json:"status-code,omitempty"` - TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty"` - CSPData *httpx.CSPData `json:"csp,omitempty"` - VHost bool `json:"vhost,omitempty"` - WebSocket bool `json:"websocket,omitempty"` - Pipeline bool `json:"pipeline,omitempty"` - HTTP2 bool `json:"http2,omitempty"` - CDN bool `json:"cdn,omitempty"` - ResponseTime string `json:"response-time,omitempty"` - Technologies []string `json:"technologies,omitempty"` - Chain []httpx.ChainItem `json:"chain,omitempty"` - FinalURL string `json:"final-url,omitempty"` - Failed bool `json:"failed"` + Error string `json:"error,omitempty" csv:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver"` + ResponseBody string `json:"response-body,omitempty" csv:"response-body"` + ContentType string `json:"content-type,omitempty" csv:"content-type"` + Method string `json:"method,omitempty" csv:"method"` + Host string `json:"host,omitempty" csv:"host"` + ContentLength int `json:"content-length,omitempty" csv:"content-length"` + ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` + StatusCode int `json:"status-code,omitempty" csv:"status-code"` + TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty" csv:"tls-grab"` + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` + VHost bool `json:"vhost,omitempty" csv:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` + HTTP2 bool `json:"http2,omitempty" csv:"http2"` + CDN bool `json:"cdn,omitempty" csv:"cdn"` + ResponseTime string `json:"response-time,omitempty" csv:"response-time"` + Technologies []string `json:"technologies,omitempty" csv:"technologies"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` + FinalURL string `json:"final-url,omitempty" csv:"final-url"` + Failed bool `json:"failed" csv:"failed"` } // JSON the result @@ -1165,6 +1179,48 @@ func (r Result) JSON(scanopts *scanOptions) string { //nolint return "" } +// CSVHeader the CSV headers +func (r Result) CSVHeader() string { //nolint + var header []string + ty := reflect.TypeOf(r) + for i := 0; i < ty.NumField(); i++ { + tag := ty.Field(i).Tag.Get("csv") + + if ignored := tag == ""; ignored { + continue + } + + header = append(header, tag) + } + return strings.Join(header, ",") +} + +// CSVRow the CSV Row +func (r Result) CSVRow(scanopts *scanOptions) string { //nolint + if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave { + r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave] + } + + buffer := bytes.Buffer{} + writer := csv.NewWriter(&buffer) + + var cells []string + elem := reflect.ValueOf(r) + for i := 0; i < elem.NumField(); i++ { + value := elem.Field(i) + tag := elem.Type().Field(i).Tag.Get(`csv`) + if ignored := tag == ""; ignored { + continue + } + + cells = append(cells, fmt.Sprintf("%v", value.Interface())) + } + _ = writer.Write(cells) + writer.Flush() + + return strings.TrimSpace(buffer.String()) // remove "\n" in the end +} + func (r *Runner) skipCDNPort(host string, port string) bool { // if the option is not enabled we don't skip if !r.options.ExcludeCDN { From 57b1ec1cc11a5ed6c374ce135bf61a22c8cd660c Mon Sep 17 00:00:00 2001 From: wux1an Date: Thu, 9 Sep 2021 18:59:08 +0800 Subject: [PATCH 009/355] doc: Update "httpx help menu" in README.md --- README.md | 144 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 75 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 35b9c34..94d8729 100644 --- a/README.md +++ b/README.md @@ -78,143 +78,149 @@ This will display help for the tool. Here are all the switches it supports. ``` -H value - Custom Header + Custom Header to send with request -allow value - Allow list of IP/CIDR's (file or comma separated) + Allow list of IP/CIDR's to process (file or comma separated) -body string - Content to send in body with HTTP request + Content to send in body with HTTP request -cdn - Check if domain's ip belongs to known CDN (akamai, cloudflare, ..) + Diplay CDN -cname - Output first cname + Display Host cname -content-length - Extracts content length + Display HTTP response content length -content-type - Extracts content-type + Display content-type header -csp-probe - Send HTTP probes on the extracted CSP domains + Send HTTP probes on the extracted CSP domains + -csv + Display output in CSV format -debug - Debug mode + Debug mode -deny value - Deny list of IP/CIDR's to process (file or comma separated) + Deny list of IP/CIDR's to process (file or comma separated) -exclude-cdn - Skip full port scans for CDNs (only checks for 80,443) + Skip full port scans for CDNs (only checks for 80,443) -extract-regex string - Extract Regex + Display response content with matched regex -fc string - Filter status code + Filter response with specific status code (-fc 403,401) -filter-regex string - Filter Regex + Filter response with specific regex -filter-string string - Filter String + Filter response with specific string -fl string - Filter content length + Filter response with specific content length (-fl 23) -follow-host-redirects - Only follow redirects on the same host + Only Follow redirects on the same host -follow-redirects - Follow Redirects + Follow HTTP Redirects -http-proxy string - HTTP Proxy, eg http://127.0.0.1:8080 + HTTP Proxy, eg http://127.0.0.1:8080 -http2 - HTTP2 probe + HTTP2 probe -include-chain - Show Raw HTTP Chain In Output (-json only) + Show Raw HTTP Chain In Output (-json only) -include-response - Show Raw HTTP Response In Output (-json only) + Show Raw HTTP response In Output (-json only) -ip - Output target ip + Display Host IP -json - JSON Output + Display output in JSON format -l string - File containing domains + Input file containing list of hosts to process -location - Extracts location header + Display location header -match-regex string - Match Regex + Match response with specific regex -match-string string - Match string + Match response with specific string + -max-host-error int + Max error count per host before skipping remaining path/s (default 30) + -max-redirects int + Max number of redirects to follow per host (default 10) -mc string - Match status code + Match response with specific status code (-mc 200,302) -method - Display request method + Display request method -ml string - Match content length + Match response with specific content length (-ml 102) -no-color - No Color + Disable colored output -no-fallback - If HTTPS on port 443 is successful on default configuration, probes also port 80 for HTTP + Probe both protocol (HTTPS and HTTP) -no-fallback-scheme - The tool will respect and attempt the scheme specified in the url (if HTTPS is specified no HTTP is attempted) + Probe with input protocol scheme -o string - File to write output to (optional) + File to write output to (optional) -path string - Request path/file (example '/api') + Request path/file (example '/api') -paths string - Command separated paths or file containing one path per line (example '/api/v1,/apiv2') + Command separated paths or file containing one path per line (example '/api/v1,/apiv2') -pipeline - HTTP1.1 Pipeline + HTTP1.1 Pipeline probe -ports value - ports range (nmap syntax: eg 1,2-10,11) + Port ranges to scan (nmap syntax: eg 1,2-10,11) -probe - Display probe status + Display probe status -random-agent - Use randomly selected HTTP User-Agent header value (default true) + Use randomly selected HTTP User-Agent header value (default true) -rate-limit int - Maximum requests to send per second (default 150) + Maximum requests to send per second (default 150) -request string - File containing raw request + File containing raw request -response-in-json - Show Raw HTTP Response In Output (-json only) (deprecated) + Show Raw HTTP response In Output (-json only) (deprecated) -response-size-to-read int - Max response size to read in bytes (default - unlimited) + Max response size to read in bytes (default - unlimited) (default 2147483647) -response-size-to-save int - Max response size to save in bytes (default - unlimited) + Max response size to save in bytes (default - unlimited) (default 2147483647) -response-time - Output the response time + Display the response time -resume - Resume scan using resume.cfg + Resume scan using resume.cfg -retries int - Number of retries + Number of retries -silent - Silent mode + Silent mode -sr - Save response to file (default 'output') + Store HTTP response to directoy (default 'output') -srd string - Save response directory (default "output") + Custom directory to store HTTP responses (default "output") -stats - Enable statistic on keypress (terminal may become unresponsive till the end) + Enable statistic on keypress (terminal may become unresponsive till the end) -status-code - Extracts status code + Display HTTP response status code -store-chain - Save chain to file (default 'output') + Save chain to file (default 'output') -tech-detect - Perform wappalyzer based technology detection + Perform wappalyzer based technology detection -threads int - Number of threads (default 50) + Number of threads (default 50) -timeout int - Timeout in seconds (default 5) + Timeout in seconds (default 5) -title - Extracts title + Display page title -tls-grab - Perform TLS data grabbing + Perform TLS(SSL) data grabbing -tls-probe - Send HTTP probes on the extracted TLS domains + Send HTTP probes on the extracted TLS domains -unsafe - Send raw requests skipping golang normalization + Send raw requests skipping golang normalization -verbose - Verbose Mode + Verbose Mode -version - Show version of httpx + Show version of httpx -vhost - Check for VHOSTs + Check for VHOSTs -vhost-input - Get a list of vhosts as input + Get a list of vhosts as input -web-server - Extracts server header + Display server header -websocket - Prints out if the server exposes a websocket + Display server using websocket -x string - Request Methods, use ALL to check all verbs () + Request Methods to use, use 'all' to probe all HTTP methods ``` From 82efbb8e6b770f71b795ce10b3380c820e6816c9 Mon Sep 17 00:00:00 2001 From: wux1an Date: Fri, 10 Sep 2021 19:49:13 +0800 Subject: [PATCH 010/355] feat: Escape ',' in CSV headers --- runner/runner.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 2980271..307835f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1181,7 +1181,10 @@ func (r Result) JSON(scanopts *scanOptions) string { //nolint // CSVHeader the CSV headers func (r Result) CSVHeader() string { //nolint - var header []string + buffer := bytes.Buffer{} + writer := csv.NewWriter(&buffer) + + var headers []string ty := reflect.TypeOf(r) for i := 0; i < ty.NumField(); i++ { tag := ty.Field(i).Tag.Get("csv") @@ -1190,9 +1193,12 @@ func (r Result) CSVHeader() string { //nolint continue } - header = append(header, tag) + headers = append(headers, tag) } - return strings.Join(header, ",") + _ = writer.Write(headers) + writer.Flush() + + return strings.TrimSpace(buffer.String()) } // CSVRow the CSV Row From 9a0dee9f08d20b7386f1dac0919b766241cb8054 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 10:04:33 +0000 Subject: [PATCH 011/355] chore(deps): bump golang from 1.17.0-alpine to 1.17.1-alpine Bumps golang from 1.17.0-alpine to 1.17.1-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1886ef6..4b7b06f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.0-alpine AS builder +FROM golang:1.17.1-alpine AS builder RUN apk add --no-cache git RUN GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx From 5f6d17c6be7a098051fa5ec2a2041d80cbb82408 Mon Sep 17 00:00:00 2001 From: wux1an Date: Mon, 13 Sep 2021 22:57:17 +0800 Subject: [PATCH 012/355] feat: Defense against csv injection --- runner/runner.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 307835f..9c10df5 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1219,7 +1219,15 @@ func (r Result) CSVRow(scanopts *scanOptions) string { //nolint continue } - cells = append(cells, fmt.Sprintf("%v", value.Interface())) + str := fmt.Sprintf("%v", value.Interface()) + + // defense against csv injection + startWithRiskyChar, _ := regexp.Compile("^([=+\\-@])") + if startWithRiskyChar.Match([]byte(str)) { + str = "'" + str + } + + cells = append(cells, str) } _ = writer.Write(cells) writer.Flush() From 997c435ee8130e3ae3e86caa2acfefaa64b511ff Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 13 Sep 2021 14:12:25 -0500 Subject: [PATCH 013/355] status-code with unsafe flag fix --- runner/runner.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index ae10f42..0e12152 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -795,26 +795,32 @@ retry: if scanopts.OutputStatusCode { builder.WriteString(" [") - for i, chainItem := range resp.Chain { + setColor := func(statusCode int) { if !scanopts.OutputWithNoColor { // Color the status code based on its value switch { - case chainItem.StatusCode >= http.StatusOK && chainItem.StatusCode < http.StatusMultipleChoices: - builder.WriteString(aurora.Green(strconv.Itoa(chainItem.StatusCode)).String()) - case chainItem.StatusCode >= http.StatusMultipleChoices && chainItem.StatusCode < http.StatusBadRequest: - builder.WriteString(aurora.Yellow(strconv.Itoa(chainItem.StatusCode)).String()) - case chainItem.StatusCode >= http.StatusBadRequest && chainItem.StatusCode < http.StatusInternalServerError: - builder.WriteString(aurora.Red(strconv.Itoa(chainItem.StatusCode)).String()) + case statusCode >= http.StatusOK && statusCode < http.StatusMultipleChoices: + builder.WriteString(aurora.Green(strconv.Itoa(statusCode)).String()) + case statusCode >= http.StatusMultipleChoices && statusCode < http.StatusBadRequest: + builder.WriteString(aurora.Yellow(strconv.Itoa(statusCode)).String()) + case statusCode >= http.StatusBadRequest && statusCode < http.StatusInternalServerError: + builder.WriteString(aurora.Red(strconv.Itoa(statusCode)).String()) case resp.StatusCode > http.StatusInternalServerError: - builder.WriteString(aurora.Bold(aurora.Yellow(strconv.Itoa(chainItem.StatusCode))).String()) + builder.WriteString(aurora.Bold(aurora.Yellow(strconv.Itoa(statusCode))).String()) } } else { - builder.WriteString(strconv.Itoa(chainItem.StatusCode)) + builder.WriteString(strconv.Itoa(statusCode)) } + } + for i, chainItem := range resp.Chain { + setColor(chainItem.StatusCode) if i != len(resp.Chain)-1 { builder.WriteRune(',') } } + if r.options.Unsafe { + setColor(resp.StatusCode) + } builder.WriteRune(']') } From 8e5f500b06f21c4000b919fa7ed2be8d7d18671e Mon Sep 17 00:00:00 2001 From: wux1an Date: Tue, 14 Sep 2021 19:08:30 +0800 Subject: [PATCH 014/355] fix: fix golangci-lint error: S1007: should use raw string (`...`) with regexp.Compile to avoid having to escape twice (gosimple) --- runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 9c10df5..57244b2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1222,7 +1222,7 @@ func (r Result) CSVRow(scanopts *scanOptions) string { //nolint str := fmt.Sprintf("%v", value.Interface()) // defense against csv injection - startWithRiskyChar, _ := regexp.Compile("^([=+\\-@])") + startWithRiskyChar, _ := regexp.Compile(`^([=+\-@])`) if startWithRiskyChar.Match([]byte(str)) { str = "'" + str } From ad559f308eb37c443ee5cb00a7a39f8d970cc16b Mon Sep 17 00:00:00 2001 From: sullo Date: Tue, 14 Sep 2021 13:06:35 -0400 Subject: [PATCH 015/355] logger error message updates: - missing word in runner.go file errors - various consistency cleanup. --- common/customports/customport.go | 12 ++++++------ runner/banner.go | 2 +- runner/options.go | 4 ++-- runner/runner.go | 18 +++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/common/customports/customport.go b/common/customports/customport.go index 7728b78..8c5a025 100644 --- a/common/customports/customport.go +++ b/common/customports/customport.go @@ -60,7 +60,7 @@ func (c *CustomPorts) Set(value string) error { } Ports[p] = protocol } else { - gologger.Warning().Msgf("Could not cast port to integer, your value: %s, resulting error %s. Skipping it\n", + gologger.Warning().Msgf("Could not cast port to integer from your value: %s. Resulting error: %s. Skipping it.\n", potentialPort, err.Error()) } } else { @@ -68,21 +68,21 @@ func (c *CustomPorts) Set(value string) error { var lowP, highP int lowP, err := strconv.Atoi(potentialRange[0]) if err != nil { - gologger.Warning().Msgf("Could not cast first port of your port range(%s) to integer, your value: %s, resulting error %s. Skipping it\n", + gologger.Warning().Msgf("Could not cast first port of your range(%s) to integer from your value: %s. Resulting error: %s. Skipping it.\n", potentialPort, potentialRange[0], err.Error()) continue } highP, err = strconv.Atoi(potentialRange[1]) if err != nil { - gologger.Warning().Msgf("Could not cast last port of your port range(%s) to integer, "+ - "your value: %s, resulting error %s. Skipping it\n", + gologger.Warning().Msgf("Could not cast last port of your port range(%s) to integer from "+ + "your value: %s. Resulting error %s. Skipping it\n", potentialPort, potentialRange[1], err.Error()) continue } if lowP > highP { - gologger.Warning().Msgf("first value of port range should be lower than the last part port "+ - "in that range, your range: [%d, %d]. Skipping it\n", + gologger.Warning().Msgf("First value of port range should be lower than the last port "+ + "from your range: [%d, %d]. Skipping it.\n", lowP, highP) continue } diff --git a/runner/banner.go b/runner/banner.go index 2beadea..9c361bc 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -19,6 +19,6 @@ func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - gologger.Print().Msgf("Use with caution. You are responsible for your actions\n") + gologger.Print().Msgf("Use with caution. You are responsible for your actions.\n") gologger.Print().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") } diff --git a/runner/options.go b/runner/options.go index f0678b5..21f5501 100644 --- a/runner/options.go +++ b/runner/options.go @@ -292,11 +292,11 @@ func ParseOptions() *Options { func (options *Options) validateOptions() { if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { - gologger.Fatal().Msgf("File %s does not exist!\n", options.InputFile) + gologger.Fatal().Msgf("File %s does not exist.\n", options.InputFile) } if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) { - gologger.Fatal().Msgf("File %s does not exist!\n", options.InputRawRequest) + gologger.Fatal().Msgf("File %s does not exist.\n", options.InputRawRequest) } var err error diff --git a/runner/runner.go b/runner/runner.go index 0e12152..d71fa63 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -135,7 +135,7 @@ func New(options *Options) (*Runner, error) { var rawRequest []byte rawRequest, err = ioutil.ReadFile(options.InputRawRequest) if err != nil { - gologger.Fatal().Msgf("Could not read raw request from '%s': %s\n", options.InputRawRequest, err) + gologger.Fatal().Msgf("Could not read raw request from path '%s': %s\n", options.InputRawRequest, err) } rrMethod, rrPath, rrHeaders, rrBody, errParse := httputilz.ParseRequest(string(rawRequest), options.Unsafe) @@ -265,11 +265,11 @@ func (r *Runner) prepareInput() { if fileutil.FileExists(r.options.InputFile) { finput, err := os.Open(r.options.InputFile) if err != nil { - gologger.Fatal().Msgf("Could read input file '%s': %s\n", r.options.InputFile, err) + gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) } numHosts, err = r.loadAndCloseFile(finput) if err != nil { - gologger.Fatal().Msgf("Could read input file '%s': %s\n", r.options.InputFile, err) + gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) } } else if r.options.InputFile != "" { files, err := fileutilz.ListFilesWithPattern(r.options.InputFile) @@ -279,11 +279,11 @@ func (r *Runner) prepareInput() { for _, file := range files { finput, err := os.Open(file) if err != nil { - gologger.Fatal().Msgf("Could read input file '%s': %s\n", r.options.InputFile, err) + gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) } numTargetsFile, err := r.loadAndCloseFile(finput) if err != nil { - gologger.Fatal().Msgf("Could read input file '%s': %s\n", r.options.InputFile, err) + gologger.Fatal().Msgf("Could not read input file '%s': %s\n", r.options.InputFile, err) } numHosts += numTargetsFile } @@ -291,7 +291,7 @@ func (r *Runner) prepareInput() { if fileutil.HasStdin() { numTargetsStdin, err := r.loadAndCloseFile(os.Stdin) if err != nil { - gologger.Fatal().Msgf("Could read input from stdin: %s\n", err) + gologger.Fatal().Msgf("Could not read input from stdin: %s\n", err) } numHosts += numTargetsStdin } @@ -303,7 +303,7 @@ func (r *Runner) prepareInput() { r.stats.AddCounter("requests", 0) err := r.stats.Start(makePrintCallback(), time.Duration(statsDisplayInterval)*time.Second) if err != nil { - gologger.Warning().Msgf("Could not create statistic: %s\n", err) + gologger.Warning().Msgf("Could not create statistics: %s\n", err) } } } @@ -1026,14 +1026,14 @@ retry: } writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644) if writeErr != nil { - gologger.Warning().Msgf("Could not write response, at path '%s', to disk: %s", responsePath, writeErr) + gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } if scanopts.StoreChain && resp.HasChain() { domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile) writeErr := ioutil.WriteFile(responsePath, []byte(resp.GetChain()), 0644) if writeErr != nil { - gologger.Warning().Msgf("Could not write response, at path '%s', to disk: %s", responsePath, writeErr) + gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } } } From 2cffea48835ac2c02baa6ce5a1b89d8432dd7e72 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 15 Sep 2021 17:40:03 -0500 Subject: [PATCH 016/355] bug in sending post request fix --- runner/runner.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 40246ed..37506d2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -700,7 +701,7 @@ retry: // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) - req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) } else { req.ContentLength = 0 req.Body = nil @@ -712,11 +713,9 @@ retry: if scanopts.Unsafe { req.Header.Add("Connection", "close") } - resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } - var requestDump []byte if scanopts.Unsafe { var errDump error @@ -726,8 +725,10 @@ retry: } } else { // Create a copy on the fly of the request body - bodyBytes, _ := req.BodyBytes() - req.Request.Body = ioutil.NopCloser(bytes.NewReader(bodyBytes)) + buf := new(bytes.Buffer) + buf.ReadFrom(req.Body) + bodyBytes := buf.Bytes() + req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) if errDump != nil { @@ -740,6 +741,7 @@ retry: req.Body = nil } } + resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) // fix the final output url fullURL := req.URL.String() From 7d4273befd18446c04eda06209492959b83af81e Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 16 Sep 2021 16:13:37 -0500 Subject: [PATCH 017/355] bug-fix while sending post request --- runner/runner.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 37506d2..85efd84 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -726,7 +726,11 @@ retry: } else { // Create a copy on the fly of the request body buf := new(bytes.Buffer) - buf.ReadFrom(req.Body) + _, err := buf.ReadFrom(req.Body) + if err != nil { + gologger.Fatal().Msgf("Could not read from request body: %s\n", err) + } + bodyBytes := buf.Bytes() req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) var errDump error From 28fda6fa060d7c4658df319959682f160c8d8965 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Fri, 17 Sep 2021 22:55:43 -0500 Subject: [PATCH 018/355] changes to adrress failed test cases --- runner/runner.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 85efd84..1bef1c0 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -8,7 +8,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io" "io/ioutil" "net" "net/http" @@ -701,7 +700,7 @@ retry: // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) - req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) + req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) } else { req.ContentLength = 0 req.Body = nil @@ -713,6 +712,7 @@ retry: if scanopts.Unsafe { req.Header.Add("Connection", "close") } + resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) if r.options.ShowStatistics { r.stats.IncrementCounter("requests", 1) } @@ -725,14 +725,12 @@ retry: } } else { // Create a copy on the fly of the request body - buf := new(bytes.Buffer) - _, err := buf.ReadFrom(req.Body) - if err != nil { - gologger.Fatal().Msgf("Could not read from request body: %s\n", err) + var bodyBytes []byte + if scanopts.RequestBody != "" { + req.ContentLength = int64(len(scanopts.RequestBody)) + req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + bodyBytes = []byte(scanopts.RequestBody) } - - bodyBytes := buf.Bytes() - req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) if errDump != nil { @@ -745,8 +743,6 @@ retry: req.Body = nil } } - resp, err := hp.Do(req, httpx.UnsafeOptions{URIPath: reqURI}) - // fix the final output url fullURL := req.URL.String() parsedURL, _ := urlutil.Parse(fullURL) From 306aa2515be0f13d2c493287a369df56cc6b5fce Mon Sep 17 00:00:00 2001 From: sullo Date: Mon, 20 Sep 2021 13:05:39 -0400 Subject: [PATCH 019/355] Typo fixes --- README.md | 2 +- common/httpx/httpx.go | 2 +- runner/options.go | 4 ++-- runner/runner.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 35b9c34..9235955 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi - Simple and modular code base making it easy to contribute. - - Fast And fully configurable flags to probe mutiple elements. + - Fast And fully configurable flags to probe multiple elements. - Supports multiple HTTP based probings. - Smart auto fallback from https to http as default. - Supports hosts, URLs and CIDR as input. diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index ca255fd..52491db 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -73,7 +73,7 @@ func New(options *Options) (*HTTPX, error) { if httpx.Options.FollowHostRedirects { // Only follow redirects on the same host up to a maximum number redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { - // Check if we get a redirect to a differen host + // Check if we get a redirect to a different host var newHost = redirectedRequest.URL.Host var oldHost = previousRequests[0].URL.Host if newHost != oldHost { diff --git a/runner/options.go b/runner/options.go index f0678b5..4dc57da 100644 --- a/runner/options.go +++ b/runner/options.go @@ -210,7 +210,7 @@ func ParseOptions() *Options { flag.Var(&options.CustomHeaders, "H", "Custom Header to send with request") flag.Var(&options.CustomPorts, "ports", "Port ranges to scan (nmap syntax: eg 1,2-10,11)") flag.BoolVar(&options.ContentLength, "content-length", false, "Display HTTP response content length") - flag.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directoy (default 'output')") + flag.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directory (default 'output')") flag.StringVar(&options.StoreResponseDir, "srd", "output", "Custom directory to store HTTP responses") flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow HTTP Redirects") flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only Follow redirects on the same host") @@ -250,7 +250,7 @@ func ParseOptions() *Options { flag.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter response with specific regex") flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match response with specific regex") flag.BoolVar(&options.OutputCName, "cname", false, "Display Host cname") - flag.BoolVar(&options.OutputCDN, "cdn", false, "Diplay CDN") + flag.BoolVar(&options.OutputCDN, "cdn", false, "Display CDN") flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Display the response time") flag.BoolVar(&options.NoFallback, "no-fallback", false, "Probe both protocol (HTTPS and HTTP)") flag.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "Probe with input protocol scheme") diff --git a/runner/runner.go b/runner/runner.go index 0e12152..2852808 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -406,7 +406,7 @@ func (r *Runner) Close() { // RunEnumeration on targets for httpx client func (r *Runner) RunEnumeration() { - // Try to create output folder if it doesnt exist + // Try to create output folder if it doesn't exist if r.options.StoreResponse && !fileutil.FolderExists(r.options.StoreResponseDir) { if err := os.MkdirAll(r.options.StoreResponseDir, os.ModePerm); err != nil { gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", r.options.StoreResponseDir, err) From f1c3474327b1ee496b78e4a28ec675aff49b682b Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 20 Sep 2021 22:44:16 -0500 Subject: [PATCH 020/355] remove bodybytes --- runner/runner.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 1bef1c0..cba318d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -725,11 +725,9 @@ retry: } } else { // Create a copy on the fly of the request body - var bodyBytes []byte if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) - bodyBytes = []byte(scanopts.RequestBody) } var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) @@ -738,7 +736,7 @@ retry: } // The original req.Body gets modified indirectly by httputil.DumpRequestOut so we set it again to nil if it was empty // Otherwise redirects like 307/308 would fail (as they require the body to be sent along) - if len(bodyBytes) == 0 { + if len(scanopts.RequestBody) == 0 { req.ContentLength = 0 req.Body = nil } From a39845bd3961112ecb235d44b3a7882d47dabb6c Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 23 Sep 2021 13:09:47 -0500 Subject: [PATCH 021/355] Added regression test for POST request with body --- cmd/integration-test/http.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 17103bf..460b087 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "io/ioutil" "net/http" "net/http/httptest" "strings" @@ -20,6 +21,7 @@ var httpTestcases = map[string]testutils.TestCase{ "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body + "Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body } type standardHttpGet struct { @@ -184,3 +186,26 @@ func (h *issue363) Execute() error { } return nil } + +type issue400 struct{} + +func (h *issue400) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.POST("/receive", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + data, _ := ioutil.ReadAll(r.Body) + fmt.Fprintf(w, "data received %s", data) + })) + ts = httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL+"/receive", debug, "-body 'a=b'", "-x POST", "-status-code") + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} From fa455e73630804c4c5ff0596703166360deed1a6 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 26 Sep 2021 06:49:49 +0200 Subject: [PATCH 022/355] Adding stream mode support --- go.mod | 2 +- go.sum | 6 +-- runner/options.go | 4 ++ runner/runner.go | 115 +++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 110 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index b757876..09ed5de 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca + github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa diff --git a/go.sum b/go.sum index 7d1bb01..b01d244 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,8 @@ github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 h1: github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= -github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//ApxoeQRbt9GgL/122688bhGy9hur8YG0Qh69k5I= -github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21 h1:YtKqlzdhAfXbV6R7QrcoaP21+lpb6G++GEDqJbeEmQA= +github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= @@ -183,8 +183,6 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= -github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe h1:tQTgf5XLBgZbkJDPtnV3SfdP9tzz5ZWeDBwv8WhnH9Q= -github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= diff --git a/runner/options.go b/runner/options.go index a532d1a..60f3420 100644 --- a/runner/options.go +++ b/runner/options.go @@ -190,6 +190,8 @@ type Options struct { resumeCfg *ResumeCfg ExcludeCDN bool HostMaxErrors int + Stream bool + SkipDedupe bool } // ParseOptions parses the command line options for application @@ -269,6 +271,8 @@ func ParseOptions() *Options { flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") flag.IntVar(&options.HostMaxErrors, "max-host-error", 30, "Max error count per host before skipping remaining path/s") + flag.BoolVar(&options.Stream, "stream", false, "Stream mode - start elaborating without sorting the input") + flag.BoolVar(&options.SkipDedupe, "skip-dedupe", false, "Don't dedupe input items (only used with stream mode)") flag.Parse() diff --git a/runner/runner.go b/runner/runner.go index 7dcb503..f0e56f0 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -232,6 +232,8 @@ func New(options *Options) (*Runner, error) { } } + hmapOptions := hybrid.DefaultDiskOptions + hmapOptions.DBType = hybrid.PogrebDB hm, err := hybrid.New(hybrid.DefaultDiskOptions) if err != nil { return nil, err @@ -310,6 +312,77 @@ func (r *Runner) prepareInput() { } } +func (r *Runner) setSeen(k string) { + r.hm.Set(k, nil) +} + +func (r *Runner) seen(k string) bool { + _, ok := r.hm.Get(k) + return ok +} + +func (r *Runner) testAndSet(k string) bool { + // skip empty lines + k = strings.TrimSpace(k) + if k == "" { + return false + } + + if r.seen(k) { + return false + } + + r.setSeen(k) + return true +} + +func (r *Runner) streamInput() (chan string, error) { + out := make(chan string) + go func() { + defer close(out) + + if fileutil.FileExists(r.options.InputFile) { + fchan, err := fileutil.ReadFile(r.options.InputFile) + if err != nil { + return + } + for item := range fchan { + if r.options.SkipDedupe || r.testAndSet(item) { + out <- item + } + } + } else if r.options.InputFile != "" { + files, err := fileutilz.ListFilesWithPattern(r.options.InputFile) + if err != nil { + gologger.Fatal().Msgf("No input provided: %s", err) + } + for _, file := range files { + fchan, err := fileutil.ReadFile(file) + if err != nil { + return + } + for item := range fchan { + if r.options.SkipDedupe || r.testAndSet(item) { + out <- item + } + } + } + } + if fileutil.HasStdin() { + fchan, err := fileutil.ReadFileWithReader(os.Stdin) + if err != nil { + return + } + for item := range fchan { + if r.options.SkipDedupe || r.testAndSet(item) { + out <- item + } + } + } + }() + return out, nil +} + func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { scanner := bufio.NewScanner(finput) for scanner.Scan() { @@ -415,11 +488,20 @@ func (r *Runner) RunEnumeration() { } } - r.prepareInput() + var streamChan chan string + if r.options.Stream { + var err error + streamChan, err = r.streamInput() + if err != nil { + gologger.Fatal().Msgf("Could not stream input: %s\n", err) + } + } else { + r.prepareInput() - // if resume is enabled inform the user - if r.options.ShouldLoadResume() && r.options.resumeCfg.Index > 0 { - gologger.Debug().Msgf("Resuming at position %d: %s\n", r.options.resumeCfg.Index, r.options.resumeCfg.ResumeFrom) + // if resume is enabled inform the user + if r.options.ShouldLoadResume() && r.options.resumeCfg.Index > 0 { + gologger.Debug().Msgf("Resuming at position %d: %s\n", r.options.resumeCfg.Index, r.options.resumeCfg.ResumeFrom) + } } // output routine @@ -498,11 +580,9 @@ func (r *Runner) RunEnumeration() { wg := sizedwaitgroup.New(r.options.Threads) - r.hm.Scan(func(k, _ []byte) error { - t := string(k) - + processItem := func(k string) error { if r.options.resumeCfg != nil { - r.options.resumeCfg.current = t + r.options.resumeCfg.current = k r.options.resumeCfg.currentIndex++ if r.options.resumeCfg.currentIndex <= r.options.resumeCfg.Index { return nil @@ -511,7 +591,7 @@ func (r *Runner) RunEnumeration() { protocol := r.options.protocol // attempt to parse url as is - if u, err := url.Parse(t); err == nil { + if u, err := url.Parse(k); err == nil { if r.options.NoFallbackScheme && u.Scheme == httpx.HTTP || u.Scheme == httpx.HTTPS { protocol = u.Scheme } @@ -521,14 +601,25 @@ func (r *Runner) RunEnumeration() { for _, p := range r.options.requestURIs { scanopts := r.scanopts.Clone() scanopts.RequestURI = p - r.process(t, &wg, r.hp, protocol, scanopts, output) + r.process(k, &wg, r.hp, protocol, scanopts, output) } } else { - r.process(t, &wg, r.hp, protocol, &r.scanopts, output) + r.process(k, &wg, r.hp, protocol, &r.scanopts, output) } return nil - }) + } + + if r.options.Stream { + for item := range streamChan { + processItem(item) + } + } else { + r.hm.Scan(func(k, _ []byte) error { + processItem(string(k)) + return nil + }) + } wg.Wait() From 3a4cf810f01a988bb48d85c18490a2a6cc99e9f1 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 26 Sep 2021 06:54:24 +0200 Subject: [PATCH 023/355] misc --- runner/runner.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index f0e56f0..f1fe054 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -313,7 +313,7 @@ func (r *Runner) prepareInput() { } func (r *Runner) setSeen(k string) { - r.hm.Set(k, nil) + _ = r.hm.Set(k, nil) } func (r *Runner) seen(k string) bool { @@ -612,12 +612,11 @@ func (r *Runner) RunEnumeration() { if r.options.Stream { for item := range streamChan { - processItem(item) + _ = processItem(item) } } else { r.hm.Scan(func(k, _ []byte) error { - processItem(string(k)) - return nil + return processItem(string(k)) }) } From aa7693da0a03f7223179ce0869353a2f0101ba80 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 29 Sep 2021 13:43:54 -0500 Subject: [PATCH 024/355] added goflags support --- README.md | 228 +++++++++++++++++----------------------------- go.mod | 3 + go.sum | 4 + runner/options.go | 171 +++++++++++++++++++--------------- 4 files changed, 187 insertions(+), 219 deletions(-) diff --git a/README.md b/README.md index 55e7773..a8ebb80 100644 --- a/README.md +++ b/README.md @@ -77,150 +77,90 @@ This will display help for the tool. Here are all the switches it supports. 👉 httpx help menu 👈 ``` - -H value - Custom Header to send with request - -allow value - Allow list of IP/CIDR's to process (file or comma separated) - -body string - Content to send in body with HTTP request - -cdn - Diplay CDN - -cname - Display Host cname - -content-length - Display HTTP response content length - -content-type - Display content-type header - -csp-probe - Send HTTP probes on the extracted CSP domains - -csv - Display output in CSV format - -debug - Debug mode - -deny value - Deny list of IP/CIDR's to process (file or comma separated) - -exclude-cdn - Skip full port scans for CDNs (only checks for 80,443) - -extract-regex string - Display response content with matched regex - -fc string - Filter response with specific status code (-fc 403,401) - -filter-regex string - Filter response with specific regex - -filter-string string - Filter response with specific string - -fl string - Filter response with specific content length (-fl 23) - -follow-host-redirects - Only Follow redirects on the same host - -follow-redirects - Follow HTTP Redirects - -http-proxy string - HTTP Proxy, eg http://127.0.0.1:8080 - -http2 - HTTP2 probe - -include-chain - Show Raw HTTP Chain In Output (-json only) - -include-response - Show Raw HTTP response In Output (-json only) - -ip - Display Host IP - -json - Display output in JSON format - -l string - Input file containing list of hosts to process - -location - Display location header - -match-regex string - Match response with specific regex - -match-string string - Match response with specific string - -max-host-error int - Max error count per host before skipping remaining path/s (default 30) - -max-redirects int - Max number of redirects to follow per host (default 10) - -mc string - Match response with specific status code (-mc 200,302) - -method - Display request method - -ml string - Match response with specific content length (-ml 102) - -no-color - Disable colored output - -no-fallback - Probe both protocol (HTTPS and HTTP) - -no-fallback-scheme - Probe with input protocol scheme - -o string - File to write output to (optional) - -path string - Request path/file (example '/api') - -paths string - Command separated paths or file containing one path per line (example '/api/v1,/apiv2') - -pipeline - HTTP1.1 Pipeline probe - -ports value - Port ranges to scan (nmap syntax: eg 1,2-10,11) - -probe - Display probe status - -random-agent - Use randomly selected HTTP User-Agent header value (default true) - -rate-limit int - Maximum requests to send per second (default 150) - -request string - File containing raw request - -response-in-json - Show Raw HTTP response In Output (-json only) (deprecated) - -response-size-to-read int - Max response size to read in bytes (default - unlimited) (default 2147483647) - -response-size-to-save int - Max response size to save in bytes (default - unlimited) (default 2147483647) - -response-time - Display the response time - -resume - Resume scan using resume.cfg - -retries int - Number of retries - -silent - Silent mode - -sr - Store HTTP response to directoy (default 'output') - -srd string - Custom directory to store HTTP responses (default "output") - -stats - Enable statistic on keypress (terminal may become unresponsive till the end) - -status-code - Display HTTP response status code - -store-chain - Save chain to file (default 'output') - -tech-detect - Perform wappalyzer based technology detection - -threads int - Number of threads (default 50) - -timeout int - Timeout in seconds (default 5) - -title - Display page title - -tls-grab - Perform TLS(SSL) data grabbing - -tls-probe - Send HTTP probes on the extracted TLS domains - -unsafe - Send raw requests skipping golang normalization - -verbose - Verbose Mode - -version - Show version of httpx - -vhost - Check for VHOSTs - -vhost-input - Get a list of vhosts as input - -web-server - Display server header - -websocket - Display server using websocket - -x string - Request Methods to use, use 'all' to probe all HTTP methods +Usage: + httpx [flags] + Flags: +TARGET: + -vhost-input Get a list of vhosts as input + -H string[] Custom Header to send with request + -ports string[] Port ranges to scan (nmap syntax: eg 1,2-10,11) + -http-proxy string HTTP Proxy, eg http://127.0.0.1:8080 + -l string Input file containing list of hosts to process + -x string Request Methods to use, use 'all' to probe all HTTP methods + -path string Request path/file (example '/api') + -paths string Command separated paths or file containing one path per line (example '/api/v1,/apiv2') + -body string Content to send in body with HTTP request + +TEMPLATE: + -tls-grab Perform TLS(SSL) data grabbing + -tech-detect Perform wappalyzer based technology detection + -threads int Number of threads (default 50) + -retries int Number of retries + -timeout int Timeout in seconds (default 5) + -vhost Check for VHOSTs + -follow-redirects Follow HTTP Redirects + -follow-host-redirects Only Follow redirects on the same host + -max-redirects int Max number of redirects to follow per host (default 10) + -tls-probe Send HTTP probes on the extracted TLS domains + -csp-probe Send HTTP probes on the extracted CSP domains + -unsafe Send raw requests skipping golang normalization + -pipeline HTTP1.1 Pipeline probe + -http2 HTTP2 probe + -no-fallback Probe both protocol (HTTPS and HTTP) + -no-fallback-scheme Probe with input protocol scheme + -random-agent Use randomly selected HTTP User-Agent header value (default true) + -allow string[] Allow list of IP/CIDR's to process (file or comma separated) + -deny string[] Deny list of IP/CIDR's to process (file or comma separated) + -response-size-to-save int Max response size to save in bytes (default - unlimited) (default 2147483647) + -response-size-to-read int Max response size to read in bytes (default - unlimited) (default 2147483647) + -resume Resume scan using resume.cfg + -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) + -max-host-error int Max error count per host before skipping remaining path/s (default 30) + +FILTERING: + -mc string Match response with specific status code (-mc 200,302) + -ml string Match response with specific content length (-ml 102) + -fc string Filter response with specific status code (-fc 403,401) + -fl string Filter response with specific content length (-fl 23) + -filter-string string Filter response with specific string + -match-string string Match response with specific string + -filter-regex string Filter response with specific regex + -match-regex string Match response with specific regex + -extract-regex string Display response content with matched regex + +RATE-LIMIT: + -rate-limit int Maximum requests to send per second (default 150) + +OUTPUT: + -o string File to write output to (optional) + -status-code Display HTTP response status code + -title Display page title + -location Display location header + -content-length Display HTTP response content length + -sr Store HTTP response to directory (default 'output') + -srd string Custom directory to store HTTP responses (default "output") + -json Display output in JSON format + -csv Display output in CSV format + -method Display request method + -silent Silent mode + -version Show version of httpx + -verbose Verbose Mode + -no-color Disable colored output + -web-server Display server header + -websocket Display server using websocket + -response-in-json Show Raw HTTP response In Output (-json only) (deprecated) + -include-response Show Raw HTTP response In Output (-json only) + -include-chain Show Raw HTTP Chain In Output (-json only) + -content-type Display content-type header + -ip Display Host IP + -request string File containing raw request + -debug Debug mode + -cname Display Host cname + -cdn Display CDN + -response-time Display the response time + -stats Enable statistic on keypress (terminal may become unresponsive till the end) + -store-chain Save chain to file (default 'output') + -probe Display probe status ``` diff --git a/go.mod b/go.mod index b757876..9bdc281 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect @@ -61,9 +62,11 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect + github.com/projectdiscovery/goflags v0.0.7 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 7d1bb01..6da7966 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -150,6 +152,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca h1:xT//A github.com/projectdiscovery/fileutil v0.0.0-20210804142714-ebba15fa53ca/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= +github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= +github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= diff --git a/runner/options.go b/runner/options.go index a532d1a..4624eb8 100644 --- a/runner/options.go +++ b/runner/options.go @@ -1,13 +1,13 @@ package runner import ( - "flag" "math" "os" "regexp" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" + "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" @@ -196,82 +196,96 @@ type Options struct { func ParseOptions() *Options { options := &Options{} - flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flag.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing") - flag.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection") - flag.IntVar(&options.Threads, "threads", 50, "Number of threads") - flag.IntVar(&options.Retries, "retries", 0, "Number of retries") - flag.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds") - flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") - flag.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs") - flag.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input") - flag.BoolVar(&options.ExtractTitle, "title", false, "Display page title") - flag.BoolVar(&options.StatusCode, "status-code", false, "Display HTTP response status code") - flag.BoolVar(&options.Location, "location", false, "Display location header") - flag.Var(&options.CustomHeaders, "H", "Custom Header to send with request") - flag.Var(&options.CustomPorts, "ports", "Port ranges to scan (nmap syntax: eg 1,2-10,11)") - flag.BoolVar(&options.ContentLength, "content-length", false, "Display HTTP response content length") - flag.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directory (default 'output')") - flag.StringVar(&options.StoreResponseDir, "srd", "output", "Custom directory to store HTTP responses") - flag.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow HTTP Redirects") - flag.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only Follow redirects on the same host") - flag.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host") - flag.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080") - flag.BoolVar(&options.JSONOutput, "json", false, "Display output in JSON format") - flag.BoolVar(&options.CSVOutput, "csv", false, "Display output in CSV format") - flag.StringVar(&options.InputFile, "l", "", "Input file containing list of hosts to process") - flag.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods") - flag.BoolVar(&options.OutputMethod, "method", false, "Display request method") - flag.BoolVar(&options.Silent, "silent", false, "Silent mode") - flag.BoolVar(&options.Version, "version", false, "Show version of httpx") - flag.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode") - flag.BoolVar(&options.NoColor, "no-color", false, "Disable colored output") - flag.BoolVar(&options.OutputServerHeader, "web-server", false, "Display server header") - flag.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket") - flag.BoolVar(&options.responseInStdout, "response-in-json", false, "Show Raw HTTP response In Output (-json only) (deprecated)") - flag.BoolVar(&options.responseInStdout, "include-response", false, "Show Raw HTTP response In Output (-json only)") - flag.BoolVar(&options.chainInStdout, "include-chain", false, "Show Raw HTTP Chain In Output (-json only)") - flag.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains") - flag.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains") - flag.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')") - flag.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')") - flag.BoolVar(&options.OutputContentType, "content-type", false, "Display content-type header") - flag.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match response with specific status code (-mc 200,302)") - flag.StringVar(&options.OutputMatchContentLength, "ml", "", "Match response with specific content length (-ml 102)") - flag.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter response with specific status code (-fc 403,401)") - flag.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter response with specific content length (-fl 23)") - flag.StringVar(&options.InputRawRequest, "request", "", "File containing raw request") - flag.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization") - flag.StringVar(&options.RequestBody, "body", "", "Content to send in body with HTTP request") - flag.BoolVar(&options.Debug, "debug", false, "Debug mode") - flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe") - flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe") - flag.BoolVar(&options.OutputIP, "ip", false, "Display Host IP") - flag.StringVar(&options.OutputFilterString, "filter-string", "", "Filter response with specific string") - flag.StringVar(&options.OutputMatchString, "match-string", "", "Match response with specific string") - flag.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter response with specific regex") - flag.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match response with specific regex") - flag.BoolVar(&options.OutputCName, "cname", false, "Display Host cname") - flag.BoolVar(&options.OutputCDN, "cdn", false, "Display CDN") - flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Display the response time") - flag.BoolVar(&options.NoFallback, "no-fallback", false, "Probe both protocol (HTTPS and HTTP)") - flag.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "Probe with input protocol scheme") - flag.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)") - flag.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value") - flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')") - flag.Var(&options.Allow, "allow", "Allow list of IP/CIDR's to process (file or comma separated)") - flag.Var(&options.Deny, "deny", "Deny list of IP/CIDR's to process (file or comma separated)") - flag.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)") - flag.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)") - flag.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Display response content with matched regex") - flag.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second") - flag.BoolVar(&options.Probe, "probe", false, "Display probe status") - flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") - flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") - flag.IntVar(&options.HostMaxErrors, "max-host-error", 30, "Max error count per host before skipping remaining path/s") + flagSet := goflags.NewFlagSet() + flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library, it is designed to maintain the result reliability with increased threads.`) - flag.Parse() + createGroup(flagSet, "input", "Target", + flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input"), + flagSet.Var(&options.CustomHeaders, "H", "Custom Header to send with request"), + flagSet.Var(&options.CustomPorts, "ports", "Port ranges to scan (nmap syntax: eg 1,2-10,11)"), + flagSet.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080"), + flagSet.StringVar(&options.InputFile, "l", "", "Input file containing list of hosts to process"), + flagSet.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods"), + flagSet.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')"), + flagSet.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')"), + flagSet.StringVar(&options.RequestBody, "body", "", "Content to send in body with HTTP request"), + ) + createGroup(flagSet, "template", "Template", + flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"), + flagSet.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection"), + flagSet.IntVar(&options.Threads, "threads", 50, "Number of threads"), + flagSet.IntVar(&options.Retries, "retries", 0, "Number of retries"), + flagSet.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds"), + flagSet.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs"), + flagSet.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow HTTP Redirects"), + flagSet.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only Follow redirects on the same host"), + flagSet.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host"), + flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"), + flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"), + flagSet.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization"), + flagSet.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe"), + flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe"), + flagSet.BoolVar(&options.NoFallback, "no-fallback", false, "Probe both protocol (HTTPS and HTTP)"), + flagSet.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "Probe with input protocol scheme"), + flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value"), + flagSet.Var(&options.Allow, "allow", "Allow list of IP/CIDR's to process (file or comma separated)"), + flagSet.Var(&options.Deny, "deny", "Deny list of IP/CIDR's to process (file or comma separated)"), + flagSet.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)"), + flagSet.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)"), + flagSet.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg"), + flagSet.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)"), + flagSet.IntVar(&options.HostMaxErrors, "max-host-error", 30, "Max error count per host before skipping remaining path/s"), + ) + + createGroup(flagSet, "filters", "Filtering", + flagSet.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match response with specific status code (-mc 200,302)"), + flagSet.StringVar(&options.OutputMatchContentLength, "ml", "", "Match response with specific content length (-ml 102)"), + flagSet.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter response with specific status code (-fc 403,401)"), + flagSet.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter response with specific content length (-fl 23)"), + flagSet.StringVar(&options.OutputFilterString, "filter-string", "", "Filter response with specific string"), + flagSet.StringVar(&options.OutputMatchString, "match-string", "", "Match response with specific string"), + flagSet.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter response with specific regex"), + flagSet.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match response with specific regex"), + flagSet.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Display response content with matched regex"), + ) + + createGroup(flagSet, "rate-limit", "Rate-limit", + flagSet.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second"), + ) + + createGroup(flagSet, "output", "Output", + flagSet.StringVar(&options.Output, "o", "", "File to write output to (optional)"), + flagSet.BoolVar(&options.StatusCode, "status-code", false, "Display HTTP response status code"), + flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"), + flagSet.BoolVar(&options.Location, "location", false, "Display location header"), + flagSet.BoolVar(&options.ContentLength, "content-length", false, "Display HTTP response content length"), + flagSet.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directory (default 'output')"), + flagSet.StringVar(&options.StoreResponseDir, "srd", "output", "Custom directory to store HTTP responses"), + flagSet.BoolVar(&options.JSONOutput, "json", false, "Display output in JSON format"), + flagSet.BoolVar(&options.CSVOutput, "csv", false, "Display output in CSV format"), + flagSet.BoolVar(&options.OutputMethod, "method", false, "Display request method"), + flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"), + flagSet.BoolVar(&options.Version, "version", false, "Show version of httpx"), + flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode"), + flagSet.BoolVar(&options.NoColor, "no-color", false, "Disable colored output"), + flagSet.BoolVar(&options.OutputServerHeader, "web-server", false, "Display server header"), + flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket"), + flagSet.BoolVar(&options.responseInStdout, "response-in-json", false, "Show Raw HTTP response In Output (-json only) (deprecated)"), + flagSet.BoolVar(&options.responseInStdout, "include-response", false, "Show Raw HTTP response In Output (-json only)"), + flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Show Raw HTTP Chain In Output (-json only)"), + flagSet.BoolVar(&options.OutputContentType, "content-type", false, "Display content-type header"), + flagSet.BoolVar(&options.OutputIP, "ip", false, "Display Host IP"), + flagSet.StringVar(&options.InputRawRequest, "request", "", "File containing raw request"), + flagSet.BoolVar(&options.Debug, "debug", false, "Debug mode"), + flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), + flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display CDN"), + flagSet.BoolVar(&options.OutputResponseTime, "response-time", false, "Display the response time"), + flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)"), + flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')"), + flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), + ) + flagSet.Parse() // Read the inputs and configure the logging options.configureOutput() @@ -366,3 +380,10 @@ func (options *Options) ShouldLoadResume() bool { func (options *Options) ShouldSaveResume() bool { return true } + +func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { + flagSet.SetGroup(groupName, description) + for _, currentFlag := range flags { + currentFlag.Group(groupName) + } +} From 995037f9b2d0b3180b34aac175e3cc9e346a5c04 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 29 Sep 2021 13:57:52 -0500 Subject: [PATCH 025/355] lint error fix --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 4624eb8..90ac2a3 100644 --- a/runner/options.go +++ b/runner/options.go @@ -285,7 +285,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')"), flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), ) - flagSet.Parse() + _ = flagSet.Parse() // Read the inputs and configure the logging options.configureOutput() From af42eef28d6f3c3a42a5c5646d6cbfc34ccda2a2 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sat, 2 Oct 2021 06:56:30 +0530 Subject: [PATCH 026/355] misc update --- runner/options.go | 145 +++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 65 deletions(-) diff --git a/runner/options.go b/runner/options.go index 90ac2a3..b4e04c6 100644 --- a/runner/options.go +++ b/runner/options.go @@ -197,93 +197,108 @@ func ParseOptions() *Options { options := &Options{} flagSet := goflags.NewFlagSet() - flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library, it is designed to maintain the result reliability with increased threads.`) + flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library, it is designed to maintain the result reliability with increased threads.`) - createGroup(flagSet, "input", "Target", - flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input"), - flagSet.Var(&options.CustomHeaders, "H", "Custom Header to send with request"), - flagSet.Var(&options.CustomPorts, "ports", "Port ranges to scan (nmap syntax: eg 1,2-10,11)"), - flagSet.StringVar(&options.HTTPProxy, "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080"), - flagSet.StringVar(&options.InputFile, "l", "", "Input file containing list of hosts to process"), - flagSet.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods"), - flagSet.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')"), - flagSet.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')"), - flagSet.StringVar(&options.RequestBody, "body", "", "Content to send in body with HTTP request"), + createGroup(flagSet, "input", "Input", + flagSet.StringVarP(&options.InputFile,"list", "l", "", "Input file containing list of hosts to process"), + flagSet.StringVar(&options.InputRawRequest, "request", "", "File containing raw request"), ) - createGroup(flagSet, "template", "Template", - flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"), - flagSet.BoolVar(&options.TechDetect, "tech-detect", false, "Perform wappalyzer based technology detection"), - flagSet.IntVar(&options.Threads, "threads", 50, "Number of threads"), - flagSet.IntVar(&options.Retries, "retries", 0, "Number of retries"), - flagSet.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds"), - flagSet.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs"), - flagSet.BoolVar(&options.FollowRedirects, "follow-redirects", false, "Follow HTTP Redirects"), - flagSet.BoolVar(&options.FollowHostRedirects, "follow-host-redirects", false, "Only Follow redirects on the same host"), - flagSet.IntVar(&options.MaxRedirects, "max-redirects", 10, "Max number of redirects to follow per host"), + createGroup(flagSet, "Probes", "Probes", + flagSet.BoolVarP(&options.StatusCode,"status-code", "sc", false, "Display HTTP response status code"), + flagSet.BoolVarP(&options.TechDetect,"tech-detect", "tech", false, "Perform wappalyzer based technology detection"), + flagSet.BoolVarP(&options.ContentLength,"content-length", "cl", false, "Display HTTP response content length"), + flagSet.BoolVarP(&options.OutputServerHeader,"web-server","server", false, "Display server header"), + flagSet.BoolVarP(&options.OutputContentType,"content-type", "ct", false, "Display content-type header"), + flagSet.BoolVarP(&options.OutputResponseTime,"response-time", "rt", false, "Display the response time"), + flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"), + flagSet.BoolVar(&options.Location, "location", false, "Display location header"), + flagSet.BoolVar(&options.OutputMethod, "method", false, "Display request method"), + flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket"), + flagSet.BoolVar(&options.OutputIP, "ip", false, "Display Host IP"), + flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), + flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display CDN"), + flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), + ) + + createGroup(flagSet, "matchers", "Matchers", + flagSet.StringVarP(&options.OutputMatchStatusCode,"match-code", "mc", "", "Match response with specific status code (-mc 200,302)"), + flagSet.StringVarP(&options.OutputMatchContentLength,"match-length", "ml", "", "Match response with specific content length (-ml 102)"), + flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms","", "Match response with specific string"), + flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr","", "Match response with specific regex"), + flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er","", "Display response content with matched regex"), + ) + + createGroup(flagSet, "filters", "Filters", + flagSet.StringVarP(&options.OutputFilterStatusCode,"filter-code", "fc", "", "Filter response with specific status code (-fc 403,401)"), + flagSet.StringVarP(&options.OutputFilterContentLength,"filter-length", "fl", "", "Filter response with specific content length (-fl 23)"), + flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"), + flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe","", "Filter response with specific regex"), + ) + + createGroup(flagSet, "rate-limit", "Rate-Limit", + flagSet.IntVarP(&options.Threads, "threads","t", 50, "Number of threads"), + flagSet.IntVarP(&options.RateLimit,"rate-limit","rl", 150, "Maximum requests to send per second"), + ) + + createGroup(flagSet, "Misc", "Miscellaneous", flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"), + flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"), flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"), - flagSet.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization"), flagSet.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe"), flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe"), - flagSet.BoolVar(&options.NoFallback, "no-fallback", false, "Probe both protocol (HTTPS and HTTP)"), - flagSet.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "Probe with input protocol scheme"), - flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value"), + flagSet.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs"), + flagSet.VarP(&options.CustomPorts,"ports","p", "Port ranges to scan (nmap syntax: eg 1,2-10,11)"), + flagSet.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')"), + flagSet.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')"), + ) + + createGroup(flagSet, "configs", "Configurations", + flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "Max response size to save in bytes"), + flagSet.IntVarP(&options.MaxResponseBodySizeToRead,"response-size-to-read", "rstr", math.MaxInt32, "Max response size to read in bytes"), flagSet.Var(&options.Allow, "allow", "Allow list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "Deny list of IP/CIDR's to process (file or comma separated)"), - flagSet.IntVar(&options.MaxResponseBodySizeToSave, "response-size-to-save", math.MaxInt32, "Max response size to save in bytes (default - unlimited)"), - flagSet.IntVar(&options.MaxResponseBodySizeToRead, "response-size-to-read", math.MaxInt32, "Max response size to read in bytes (default - unlimited)"), + flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value"), + flagSet.VarP(&options.CustomHeaders,"header", "H", "Custom Header to send with request"), + flagSet.StringVarP(&options.HTTPProxy,"proxy", "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080"), + flagSet.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization"), flagSet.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg"), - flagSet.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)"), - flagSet.IntVar(&options.HostMaxErrors, "max-host-error", 30, "Max error count per host before skipping remaining path/s"), + flagSet.BoolVarP(&options.NoColor,"no-color", "nc", false, "Disable colored output"), + flagSet.BoolVarP(&options.NoFallback,"no-fallback", "nf", false, "Probe both protocol (HTTPS and HTTP)"), + flagSet.BoolVarP(&options.NoFallbackScheme,"no-fallback-scheme", "nfs", false, "Probe with input protocol scheme"), + flagSet.BoolVarP(&options.FollowRedirects,"follow-redirects", "fr", false, "Follow HTTP Redirects"), + flagSet.BoolVarP(&options.FollowHostRedirects,"follow-host-redirects","fhr", false, "Only Follow redirects on the same host"), + flagSet.IntVarP(&options.MaxRedirects,"max-redirects","maxr", 10, "Max number of redirects to follow per host"), + flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input"), + flagSet.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods"), + flagSet.StringVar(&options.RequestBody, "body", "", "Content to send in body with HTTP request"), ) - createGroup(flagSet, "filters", "Filtering", - flagSet.StringVar(&options.OutputMatchStatusCode, "mc", "", "Match response with specific status code (-mc 200,302)"), - flagSet.StringVar(&options.OutputMatchContentLength, "ml", "", "Match response with specific content length (-ml 102)"), - flagSet.StringVar(&options.OutputFilterStatusCode, "fc", "", "Filter response with specific status code (-fc 403,401)"), - flagSet.StringVar(&options.OutputFilterContentLength, "fl", "", "Filter response with specific content length (-fl 23)"), - flagSet.StringVar(&options.OutputFilterString, "filter-string", "", "Filter response with specific string"), - flagSet.StringVar(&options.OutputMatchString, "match-string", "", "Match response with specific string"), - flagSet.StringVar(&options.OutputFilterRegex, "filter-regex", "", "Filter response with specific regex"), - flagSet.StringVar(&options.OutputMatchRegex, "match-regex", "", "Match response with specific regex"), - flagSet.StringVar(&options.OutputExtractRegex, "extract-regex", "", "Display response content with matched regex"), + createGroup(flagSet, "Optimizations", "Optimizations", + flagSet.IntVar(&options.Retries, "retries", 0, "Number of retries"), + flagSet.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds"), + flagSet.IntVarP(&options.HostMaxErrors,"max-host-error", "maxhr", 30, "Max error count per host before skipping remaining path/s"), + flagSet.BoolVarP(&options.ExcludeCDN,"exclude-cdn", "ec", false, "Skip full port scans for CDNs (only checks for 80,443)"), ) - createGroup(flagSet, "rate-limit", "Rate-limit", - flagSet.IntVar(&options.RateLimit, "rate-limit", 150, "Maximum requests to send per second"), + createGroup(flagSet, "debug", "Debug", + flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"), + flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode"), + flagSet.BoolVar(&options.Version, "version", false, "Show version of httpx"), + flagSet.BoolVar(&options.Debug, "debug", false, "Debug mode"), + flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)"), ) createGroup(flagSet, "output", "Output", - flagSet.StringVar(&options.Output, "o", "", "File to write output to (optional)"), - flagSet.BoolVar(&options.StatusCode, "status-code", false, "Display HTTP response status code"), - flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"), - flagSet.BoolVar(&options.Location, "location", false, "Display location header"), - flagSet.BoolVar(&options.ContentLength, "content-length", false, "Display HTTP response content length"), + flagSet.StringVarP(&options.Output,"output", "o", "", "File to write output to (optional)"), flagSet.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directory (default 'output')"), flagSet.StringVar(&options.StoreResponseDir, "srd", "output", "Custom directory to store HTTP responses"), flagSet.BoolVar(&options.JSONOutput, "json", false, "Display output in JSON format"), - flagSet.BoolVar(&options.CSVOutput, "csv", false, "Display output in CSV format"), - flagSet.BoolVar(&options.OutputMethod, "method", false, "Display request method"), - flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"), - flagSet.BoolVar(&options.Version, "version", false, "Show version of httpx"), - flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode"), - flagSet.BoolVar(&options.NoColor, "no-color", false, "Disable colored output"), - flagSet.BoolVar(&options.OutputServerHeader, "web-server", false, "Display server header"), - flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket"), - flagSet.BoolVar(&options.responseInStdout, "response-in-json", false, "Show Raw HTTP response In Output (-json only) (deprecated)"), - flagSet.BoolVar(&options.responseInStdout, "include-response", false, "Show Raw HTTP response In Output (-json only)"), + flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr",false, "Show Raw HTTP response In Output (-json only)"), flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Show Raw HTTP Chain In Output (-json only)"), - flagSet.BoolVar(&options.OutputContentType, "content-type", false, "Display content-type header"), - flagSet.BoolVar(&options.OutputIP, "ip", false, "Display Host IP"), - flagSet.StringVar(&options.InputRawRequest, "request", "", "File containing raw request"), - flagSet.BoolVar(&options.Debug, "debug", false, "Debug mode"), - flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), - flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display CDN"), - flagSet.BoolVar(&options.OutputResponseTime, "response-time", false, "Display the response time"), - flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)"), flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')"), - flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), + flagSet.BoolVar(&options.CSVOutput, "csv", false, "Display output in CSV format"), + ) _ = flagSet.Parse() // Read the inputs and configure the logging From 8e09ab55e359eb30a0e7d53a411771854c9be365 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 3 Oct 2021 01:53:13 +0530 Subject: [PATCH 027/355] misc update --- runner/options.go | 78 +++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/runner/options.go b/runner/options.go index b4e04c6..e83e2de 100644 --- a/runner/options.go +++ b/runner/options.go @@ -197,7 +197,7 @@ func ParseOptions() *Options { options := &Options{} flagSet := goflags.NewFlagSet() - flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library, it is designed to maintain the result reliability with increased threads.`) + flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library.`) createGroup(flagSet, "input", "Input", flagSet.StringVarP(&options.InputFile,"list", "l", "", "Input file containing list of hosts to process"), @@ -205,33 +205,34 @@ func ParseOptions() *Options { ) createGroup(flagSet, "Probes", "Probes", - flagSet.BoolVarP(&options.StatusCode,"status-code", "sc", false, "Display HTTP response status code"), - flagSet.BoolVarP(&options.TechDetect,"tech-detect", "tech", false, "Perform wappalyzer based technology detection"), - flagSet.BoolVarP(&options.ContentLength,"content-length", "cl", false, "Display HTTP response content length"), - flagSet.BoolVarP(&options.OutputServerHeader,"web-server","server", false, "Display server header"), - flagSet.BoolVarP(&options.OutputContentType,"content-type", "ct", false, "Display content-type header"), + flagSet.BoolVarP(&options.StatusCode,"status-code", "sc", false, "Display Status Code"), + flagSet.BoolVarP(&options.TechDetect,"tech-detect", "tech", false, "Display wappalyzer based technology detection"), + flagSet.BoolVarP(&options.ContentLength,"content-length", "cl", false, "Display Content-Length"), + flagSet.BoolVarP(&options.OutputServerHeader,"web-server","server", false, "Display Server header"), + flagSet.BoolVarP(&options.OutputContentType,"content-type", "ct", false, "Display Content-Type header"), flagSet.BoolVarP(&options.OutputResponseTime,"response-time", "rt", false, "Display the response time"), flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"), - flagSet.BoolVar(&options.Location, "location", false, "Display location header"), - flagSet.BoolVar(&options.OutputMethod, "method", false, "Display request method"), + flagSet.BoolVar(&options.Location, "location", false, "Display Location header"), + flagSet.BoolVar(&options.OutputMethod, "method", false, "Display Request method"), flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket"), flagSet.BoolVar(&options.OutputIP, "ip", false, "Display Host IP"), flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), - flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display CDN"), + flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"), flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), + flagSet.BoolVarP(&options.NoFallback,"no-fallback", "nf", false, "Display both protocol (HTTPS and HTTP)"), ) createGroup(flagSet, "matchers", "Matchers", - flagSet.StringVarP(&options.OutputMatchStatusCode,"match-code", "mc", "", "Match response with specific status code (-mc 200,302)"), - flagSet.StringVarP(&options.OutputMatchContentLength,"match-length", "ml", "", "Match response with specific content length (-ml 102)"), - flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms","", "Match response with specific string"), + flagSet.StringVarP(&options.OutputMatchStatusCode,"match-code", "mc", "", "Match response with given status code (-mc 200,302)"), + flagSet.StringVarP(&options.OutputMatchContentLength,"match-length", "ml", "", "Match response with given content length (-ml 100,102)"), + flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms","", "Match response with given string"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr","", "Match response with specific regex"), flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er","", "Display response content with matched regex"), ) createGroup(flagSet, "filters", "Filters", - flagSet.StringVarP(&options.OutputFilterStatusCode,"filter-code", "fc", "", "Filter response with specific status code (-fc 403,401)"), - flagSet.StringVarP(&options.OutputFilterContentLength,"filter-length", "fl", "", "Filter response with specific content length (-fl 23)"), + flagSet.StringVarP(&options.OutputFilterStatusCode,"filter-code", "fc", "", "Filter response with given status code (-fc 403,401)"), + flagSet.StringVarP(&options.OutputFilterContentLength,"filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"), flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe","", "Filter response with specific regex"), ) @@ -242,36 +243,35 @@ func ParseOptions() *Options { ) createGroup(flagSet, "Misc", "Miscellaneous", - flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"), flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"), + flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"), flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"), flagSet.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe"), flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe"), - flagSet.BoolVar(&options.VHost, "vhost", false, "Check for VHOSTs"), - flagSet.VarP(&options.CustomPorts,"ports","p", "Port ranges to scan (nmap syntax: eg 1,2-10,11)"), - flagSet.StringVar(&options.RequestURI, "path", "", "Request path/file (example '/api')"), - flagSet.StringVar(&options.RequestURIs, "paths", "", "Command separated paths or file containing one path per line (example '/api/v1,/apiv2')"), + flagSet.BoolVar(&options.VHost, "vhost", false, "VHOST Probe"), + flagSet.VarP(&options.CustomPorts,"ports","p", "Port to scan (nmap syntax: eg 1,2-10,11)"), + flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"), + flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"), ) createGroup(flagSet, "configs", "Configurations", flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "Max response size to save in bytes"), flagSet.IntVarP(&options.MaxResponseBodySizeToRead,"response-size-to-read", "rstr", math.MaxInt32, "Max response size to read in bytes"), - flagSet.Var(&options.Allow, "allow", "Allow list of IP/CIDR's to process (file or comma separated)"), - flagSet.Var(&options.Deny, "deny", "Deny list of IP/CIDR's to process (file or comma separated)"), - flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Use randomly selected HTTP User-Agent header value"), + flagSet.Var(&options.Allow, "allow", "Allowed list of IP/CIDR's to process (file or comma separated)"), + flagSet.Var(&options.Deny, "deny", "Denied list of IP/CIDR's to process (file or comma separated)"), + flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), flagSet.VarP(&options.CustomHeaders,"header", "H", "Custom Header to send with request"), flagSet.StringVarP(&options.HTTPProxy,"proxy", "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080"), flagSet.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization"), flagSet.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg"), - flagSet.BoolVarP(&options.NoColor,"no-color", "nc", false, "Disable colored output"), - flagSet.BoolVarP(&options.NoFallback,"no-fallback", "nf", false, "Probe both protocol (HTTPS and HTTP)"), + flagSet.BoolVarP(&options.NoColor,"no-color", "nc", false, "Disable color in output"), flagSet.BoolVarP(&options.NoFallbackScheme,"no-fallback-scheme", "nfs", false, "Probe with input protocol scheme"), - flagSet.BoolVarP(&options.FollowRedirects,"follow-redirects", "fr", false, "Follow HTTP Redirects"), - flagSet.BoolVarP(&options.FollowHostRedirects,"follow-host-redirects","fhr", false, "Only Follow redirects on the same host"), + flagSet.BoolVarP(&options.FollowRedirects,"follow-redirects", "fr", false, "Follow HTTP redirects"), + flagSet.BoolVarP(&options.FollowHostRedirects,"follow-host-redirects","fhr", false, "Follow redirects on the same host"), flagSet.IntVarP(&options.MaxRedirects,"max-redirects","maxr", 10, "Max number of redirects to follow per host"), flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input"), - flagSet.StringVar(&options.Methods, "x", "", "Request Methods to use, use 'all' to probe all HTTP methods"), - flagSet.StringVar(&options.RequestBody, "body", "", "Content to send in body with HTTP request"), + flagSet.StringVar(&options.Methods, "x", "", "Request methods to use, use 'all' to probe all HTTP methods"), + flagSet.StringVar(&options.RequestBody, "body", "", "Post body to include in HTTP request"), ) createGroup(flagSet, "Optimizations", "Optimizations", @@ -283,21 +283,21 @@ func ParseOptions() *Options { createGroup(flagSet, "debug", "Debug", flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"), - flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose Mode"), - flagSet.BoolVar(&options.Version, "version", false, "Show version of httpx"), + flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose mode"), + flagSet.BoolVar(&options.Version, "version", false, "Display version"), flagSet.BoolVar(&options.Debug, "debug", false, "Debug mode"), - flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)"), + flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Display scan statistic"), ) createGroup(flagSet, "output", "Output", - flagSet.StringVarP(&options.Output,"output", "o", "", "File to write output to (optional)"), - flagSet.BoolVar(&options.StoreResponse, "sr", false, "Store HTTP response to directory (default 'output')"), - flagSet.StringVar(&options.StoreResponseDir, "srd", "output", "Custom directory to store HTTP responses"), - flagSet.BoolVar(&options.JSONOutput, "json", false, "Display output in JSON format"), - flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr",false, "Show Raw HTTP response In Output (-json only)"), - flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Show Raw HTTP Chain In Output (-json only)"), - flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')"), - flagSet.BoolVar(&options.CSVOutput, "csv", false, "Display output in CSV format"), + flagSet.StringVarP(&options.Output,"output", "o", "", "File to write output"), + flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "Store HTTP responses"), + flagSet.StringVarP(&options.StoreResponseDir,"store-response-dir", "srd", "output", "Custom directory to store HTTP responses"), + flagSet.BoolVar(&options.JSONOutput, "json", false, "Output in JSONL(ines) format"), + flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr",false, "Include HTTP request/response in JSON output (-json only)"), + flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Include redirect HTTP Chain in JSON output (-json only)"), + flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Include HTTP redirect chain in responses (-sr only)"), + flagSet.BoolVar(&options.CSVOutput, "csv", false, "Output in CSV format"), ) _ = flagSet.Parse() From 57051e1a3ed3518b983130bbe3105a6018ed15b8 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 3 Oct 2021 02:03:48 +0530 Subject: [PATCH 028/355] readme update --- README.md | 199 ++++++++++++++++++++++++---------------------- runner/options.go | 35 ++++---- 2 files changed, 121 insertions(+), 113 deletions(-) diff --git a/README.md b/README.md index a8ebb80..8481d8b 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,10 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi # Installation Instructions -httpx requires **go1.14+** to install successfully. Run the following command to get the repo - +httpx requires **go1.17** to install successfully. Run the following command to get the repo - ```sh -GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx +go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest ``` # Usage @@ -73,96 +73,103 @@ httpx -h This will display help for the tool. Here are all the switches it supports. -
- 👉 httpx help menu 👈 -``` +```console Usage: - httpx [flags] - Flags: -TARGET: - -vhost-input Get a list of vhosts as input - -H string[] Custom Header to send with request - -ports string[] Port ranges to scan (nmap syntax: eg 1,2-10,11) - -http-proxy string HTTP Proxy, eg http://127.0.0.1:8080 - -l string Input file containing list of hosts to process - -x string Request Methods to use, use 'all' to probe all HTTP methods - -path string Request path/file (example '/api') - -paths string Command separated paths or file containing one path per line (example '/api/v1,/apiv2') - -body string Content to send in body with HTTP request - -TEMPLATE: - -tls-grab Perform TLS(SSL) data grabbing - -tech-detect Perform wappalyzer based technology detection - -threads int Number of threads (default 50) - -retries int Number of retries - -timeout int Timeout in seconds (default 5) - -vhost Check for VHOSTs - -follow-redirects Follow HTTP Redirects - -follow-host-redirects Only Follow redirects on the same host - -max-redirects int Max number of redirects to follow per host (default 10) - -tls-probe Send HTTP probes on the extracted TLS domains - -csp-probe Send HTTP probes on the extracted CSP domains - -unsafe Send raw requests skipping golang normalization - -pipeline HTTP1.1 Pipeline probe - -http2 HTTP2 probe - -no-fallback Probe both protocol (HTTPS and HTTP) - -no-fallback-scheme Probe with input protocol scheme - -random-agent Use randomly selected HTTP User-Agent header value (default true) - -allow string[] Allow list of IP/CIDR's to process (file or comma separated) - -deny string[] Deny list of IP/CIDR's to process (file or comma separated) - -response-size-to-save int Max response size to save in bytes (default - unlimited) (default 2147483647) - -response-size-to-read int Max response size to read in bytes (default - unlimited) (default 2147483647) - -resume Resume scan using resume.cfg - -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) - -max-host-error int Max error count per host before skipping remaining path/s (default 30) - -FILTERING: - -mc string Match response with specific status code (-mc 200,302) - -ml string Match response with specific content length (-ml 102) - -fc string Filter response with specific status code (-fc 403,401) - -fl string Filter response with specific content length (-fl 23) - -filter-string string Filter response with specific string - -match-string string Match response with specific string - -filter-regex string Filter response with specific regex - -match-regex string Match response with specific regex - -extract-regex string Display response content with matched regex + ./httpx [flags] + +Flags: +INPUT: + -l, -list string Input file containing list of hosts to process + -request string File containing raw request + +PROBES: + -sc, -status-code Display Status Code + -tech, -tech-detect Display wappalyzer based technology detection + -cl, -content-length Display Content-Length + -server, -web-server Display Server header + -ct, -content-type Display Content-Type header + -rt, -response-time Display the response time + -title Display page title + -location Display Location header + -method Display Request method + -websocket Display server using websocket + -ip Display Host IP + -cname Display Host cname + -cdn Display if CDN in use + -probe Display probe status + -nf, -no-fallback Display both protocol (HTTPS and HTTP) + +MATCHERS: + -mc, -match-code string Match response with given status code (-mc 200,302) + -ml, -match-length string Match response with given content length (-ml 100,102) + -ms, -match-string string Match response with given string + -mr, -match-regex string Match response with specific regex + -er, -extract-regex string Display response content with matched regex + +FILTERS: + -fc, -filter-code string Filter response with given status code (-fc 403,401) + -fl, -filter-length string Filter response with given content length (-fl 23,33) + -fs, -filter-string string Filter response with specific string + -fe, -filter-regex string Filter response with specific regex RATE-LIMIT: - -rate-limit int Maximum requests to send per second (default 150) + -t, -threads int Number of threads (default 50) + -rl, -rate-limit int Maximum requests to send per second (default 150) + +MISCELLANEOUS: + -tls-grab Perform TLS(SSL) data grabbing + -tls-probe Send HTTP probes on the extracted TLS domains + -csp-probe Send HTTP probes on the extracted CSP domains + -pipeline HTTP1.1 Pipeline probe + -http2 HTTP2 probe + -vhost VHOST Probe + -p, -ports string[] Port to scan (nmap syntax: eg 1,2-10,11) + -path string File or comma separated paths to request + -paths string File or comma separated paths to request (deprecated) OUTPUT: - -o string File to write output to (optional) - -status-code Display HTTP response status code - -title Display page title - -location Display location header - -content-length Display HTTP response content length - -sr Store HTTP response to directory (default 'output') - -srd string Custom directory to store HTTP responses (default "output") - -json Display output in JSON format - -csv Display output in CSV format - -method Display request method - -silent Silent mode - -version Show version of httpx - -verbose Verbose Mode - -no-color Disable colored output - -web-server Display server header - -websocket Display server using websocket - -response-in-json Show Raw HTTP response In Output (-json only) (deprecated) - -include-response Show Raw HTTP response In Output (-json only) - -include-chain Show Raw HTTP Chain In Output (-json only) - -content-type Display content-type header - -ip Display Host IP - -request string File containing raw request - -debug Debug mode - -cname Display Host cname - -cdn Display CDN - -response-time Display the response time - -stats Enable statistic on keypress (terminal may become unresponsive till the end) - -store-chain Save chain to file (default 'output') - -probe Display probe status + -o, -output string File to write output + -sr, -store-response Store HTTP responses + -srd, -store-response-dir string Custom directory to store HTTP responses (default "output") + -json Output in JSONL(ines) format + -irr, -include-response Include HTTP request/response in JSON output (-json only) + -include-chain Include redirect HTTP Chain in JSON output (-json only) + -store-chain Include HTTP redirect chain in responses (-sr only) + -csv Output in CSV format + +CONFIGURATIONS: + -rsts, -response-size-to-save int Max response size to save in bytes (default 2147483647) + -rstr, -response-size-to-read int Max response size to read in bytes (default 2147483647) + -allow string[] Allowed list of IP/CIDR's to process (file or comma separated) + -deny string[] Denied list of IP/CIDR's to process (file or comma separated) + -random-agent Enable Random User-Agent to use (default true) + -H, -header string[] Custom Header to send with request + -http-proxy, -proxy string HTTP Proxy, eg http://127.0.0.1:8080 + -unsafe Send raw requests skipping golang normalization + -resume Resume scan using resume.cfg + -nc, -no-color Disable color in output + -nfs, -no-fallback-scheme Probe with input protocol scheme + -fr, -follow-redirects Follow HTTP redirects + -fhr, -follow-host-redirects Follow redirects on the same host + -maxr, -max-redirects int Max number of redirects to follow per host (default 10) + -vhost-input Get a list of vhosts as input + -x string Request methods to use, use 'all' to probe all HTTP methods + -body string Post body to include in HTTP request + +OPTIMIZATIONS: + -retries int Number of retries + -timeout int Timeout in seconds (default 5) + -maxhr, -max-host-error int Max error count per host before skipping remaining path/s (default 30) + -ec, -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) + +DEBUG: + -silent Silent mode + -verbose Verbose mode + -version Display version + -debug Debug mode + -stats Display scan statistic ``` -
# Running httpX @@ -170,14 +177,14 @@ OUTPUT: This will run the tool against all the hosts and subdomains in `hosts.txt` and returns URLs running HTTP webserver. -```sh -▶ cat hosts.txt | httpx +```console +cat hosts.txt | httpx __ __ __ _ __ / /_ / /_/ /_____ | |/ / / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | -/_/ /_/\__/\__/ .___/_/|_| v1.0 +/_/ /_/\__/\__/ .___/_/|_| v1.1.1 /_/ projectdiscovery.io @@ -199,8 +206,8 @@ https://support.hackerone.com This will run the tool with the `probe` flag against all of the hosts in **hosts.txt** and return URLs with probed status. -```sh -▶ httpx -l hosts.txt -silent -probe +```console +httpx -list hosts.txt -silent -probe http://ns.hackerone.com [FAILED] https://docs.hackerone.com [SUCCESS] @@ -226,8 +233,8 @@ http://b.ns.hackerone.com [FAILED] ### Running httpx with CIDR input -```sh -▶ echo 173.0.84.0/24 | httpx -silent +```console +echo 173.0.84.0/24 | httpx -silent https://173.0.84.29 https://173.0.84.43 @@ -253,15 +260,15 @@ https://173.0.84.34 ### Running httpx with subfinder -```sh -subfinder -d hackerone.com | httpx -title -tech-detect -status-code +```console +subfinder -d hackerone.com -silent| httpx -title -tech-detect -status-code __ __ __ _ __ / /_ / /_/ /_____ | |/ / / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.0.6 + /_/ v1.1.1 projectdiscovery.io @@ -285,7 +292,7 @@ https://resources.hackerone.com [301,301,404] [Sorry, no Folders found.] - When using `json` flag, all the information (default probes) included in the JSON output. -# Thanks +# Acknowledgement httpx is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/httpx/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow: diff --git a/runner/options.go b/runner/options.go index e83e2de..7b132ec 100644 --- a/runner/options.go +++ b/runner/options.go @@ -254,6 +254,18 @@ func ParseOptions() *Options { flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"), ) + createGroup(flagSet, "output", "Output", + flagSet.StringVarP(&options.Output,"output", "o", "", "File to write output"), + flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "Store HTTP responses"), + flagSet.StringVarP(&options.StoreResponseDir,"store-response-dir", "srd", "output", "Custom directory to store HTTP responses"), + flagSet.BoolVar(&options.JSONOutput, "json", false, "Output in JSONL(ines) format"), + flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr",false, "Include HTTP request/response in JSON output (-json only)"), + flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Include redirect HTTP Chain in JSON output (-json only)"), + flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Include HTTP redirect chain in responses (-sr only)"), + flagSet.BoolVar(&options.CSVOutput, "csv", false, "Output in CSV format"), + + ) + createGroup(flagSet, "configs", "Configurations", flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "Max response size to save in bytes"), flagSet.IntVarP(&options.MaxResponseBodySizeToRead,"response-size-to-read", "rstr", math.MaxInt32, "Max response size to read in bytes"), @@ -274,13 +286,6 @@ func ParseOptions() *Options { flagSet.StringVar(&options.RequestBody, "body", "", "Post body to include in HTTP request"), ) - createGroup(flagSet, "Optimizations", "Optimizations", - flagSet.IntVar(&options.Retries, "retries", 0, "Number of retries"), - flagSet.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds"), - flagSet.IntVarP(&options.HostMaxErrors,"max-host-error", "maxhr", 30, "Max error count per host before skipping remaining path/s"), - flagSet.BoolVarP(&options.ExcludeCDN,"exclude-cdn", "ec", false, "Skip full port scans for CDNs (only checks for 80,443)"), - ) - createGroup(flagSet, "debug", "Debug", flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"), flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose mode"), @@ -289,17 +294,13 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Display scan statistic"), ) - createGroup(flagSet, "output", "Output", - flagSet.StringVarP(&options.Output,"output", "o", "", "File to write output"), - flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "Store HTTP responses"), - flagSet.StringVarP(&options.StoreResponseDir,"store-response-dir", "srd", "output", "Custom directory to store HTTP responses"), - flagSet.BoolVar(&options.JSONOutput, "json", false, "Output in JSONL(ines) format"), - flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr",false, "Include HTTP request/response in JSON output (-json only)"), - flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Include redirect HTTP Chain in JSON output (-json only)"), - flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Include HTTP redirect chain in responses (-sr only)"), - flagSet.BoolVar(&options.CSVOutput, "csv", false, "Output in CSV format"), - + createGroup(flagSet, "Optimizations", "Optimizations", + flagSet.IntVar(&options.Retries, "retries", 0, "Number of retries"), + flagSet.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds"), + flagSet.IntVarP(&options.HostMaxErrors,"max-host-error", "maxhr", 30, "Max error count per host before skipping remaining path/s"), + flagSet.BoolVarP(&options.ExcludeCDN,"exclude-cdn", "ec", false, "Skip full port scans for CDNs (only checks for 80,443)"), ) + _ = flagSet.Parse() // Read the inputs and configure the logging options.configureOutput() From f13a8f3901addc4cab0104bc29b2d740984cce3c Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 3 Oct 2021 02:25:32 +0530 Subject: [PATCH 029/355] dockerhub workflow update --- .github/workflows/dockerhub-push.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 3883d79..065d143 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -1,8 +1,10 @@ name: 🌥 Docker Push on: - release: - types: [published] + workflow_run: + workflows: ["🎉 Release Binary"] + types: + - completed workflow_dispatch: jobs: From e7819fc39c583a25068ea495f2c7ae0f30d5bbe8 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 3 Oct 2021 02:37:31 +0530 Subject: [PATCH 030/355] misc update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 55e7773..e8058ef 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi # Features -

- httpx +

+ httpx

From fbe91f6778432efd1198f0ecb2be3f1252338cae Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 4 Oct 2021 16:36:19 +0530 Subject: [PATCH 031/355] Update options.go --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 60f3420..442a3ba 100644 --- a/runner/options.go +++ b/runner/options.go @@ -271,7 +271,7 @@ func ParseOptions() *Options { flag.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg") flag.BoolVar(&options.ExcludeCDN, "exclude-cdn", false, "Skip full port scans for CDNs (only checks for 80,443)") flag.IntVar(&options.HostMaxErrors, "max-host-error", 30, "Max error count per host before skipping remaining path/s") - flag.BoolVar(&options.Stream, "stream", false, "Stream mode - start elaborating without sorting the input") + flag.BoolVar(&options.Stream, "stream", false, "Stream mode - start elaborating input targets without sorting") flag.BoolVar(&options.SkipDedupe, "skip-dedupe", false, "Don't dedupe input items (only used with stream mode)") flag.Parse() From ed95845496adc2b411e81d4ee91bf1f8df89a759 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 4 Oct 2021 16:48:00 +0530 Subject: [PATCH 032/355] misc flag update --- runner/options.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runner/options.go b/runner/options.go index 7b132ec..725709b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -190,6 +190,8 @@ type Options struct { resumeCfg *ResumeCfg ExcludeCDN bool HostMaxErrors int + Stream bool + SkipDedupe bool } // ParseOptions parses the command line options for application @@ -284,6 +286,8 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input"), flagSet.StringVar(&options.Methods, "x", "", "Request methods to use, use 'all' to probe all HTTP methods"), flagSet.StringVar(&options.RequestBody, "body", "", "Post body to include in HTTP request"), + flagSet.BoolVarP(&options.Stream, "stream","s", false, "Stream mode - start elaborating input targets without sorting"), + flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe","sd", false, "Disable dedupe input items (only used with stream mode)"), ) createGroup(flagSet, "debug", "Debug", From 6ebbba594e419bfecbe9a75eb6ab5b481de94fdb Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 4 Oct 2021 16:51:25 +0530 Subject: [PATCH 033/355] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 9c361bc..0cdcb49 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.2 + /_/ v1.1.3 ` // Version is the current version of httpx -const Version = `v1.1.2` +const Version = `v1.1.3` // showBanner is used to show the banner to the user func showBanner() { From 315d06d554d624996efca7a1991b4d0d13e3a869 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 4 Oct 2021 17:17:03 +0530 Subject: [PATCH 034/355] Added docker example --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 2357b8c..2f2eb44 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,34 @@ https://support.hackerone.com [301,302,301,200] [HackerOne] [Cloudflare,Ruby on https://resources.hackerone.com [301,301,404] [Sorry, no Folders found.] ``` +### Running httpx with docker + +```console +cat sub_domains.txt | docker run -i projectdiscovery/httpx + + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ v1.1.2 + + projectdiscovery.io + +Use with caution. You are responsible for your actions +Developers assume no liability and are not responsible for any misuse or damage. +https://mta-sts.forwarding.hackerone.com +https://mta-sts.hackerone.com +https://mta-sts.managed.hackerone.com +https://www.hackerone.com +https://api.hackerone.com +https://gslink.hackerone.com +https://resources.hackerone.com +https://docs.hackerone.com +https://support.hackerone.com +``` + + # 📋 Notes - As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable. From 102c3eafffdaab5250ee7734f3305a3753eae07d Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 4 Oct 2021 17:18:23 +0530 Subject: [PATCH 035/355] docker install update --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4b7b06f..be4b7f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.17.1-alpine AS builder RUN apk add --no-cache git -RUN GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx +RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest FROM alpine:3.14 RUN apk -U upgrade --no-cache \ From f78f94dffeb0523905a2638fa51edc3c6c376635 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 11 Oct 2021 05:22:28 +0530 Subject: [PATCH 036/355] misc flag update --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 725709b..d88b002 100644 --- a/runner/options.go +++ b/runner/options.go @@ -208,7 +208,7 @@ func ParseOptions() *Options { createGroup(flagSet, "Probes", "Probes", flagSet.BoolVarP(&options.StatusCode,"status-code", "sc", false, "Display Status Code"), - flagSet.BoolVarP(&options.TechDetect,"tech-detect", "tech", false, "Display wappalyzer based technology detection"), + flagSet.BoolVarP(&options.TechDetect,"tech-detect", "td", false, "Display wappalyzer based technology detection"), flagSet.BoolVarP(&options.ContentLength,"content-length", "cl", false, "Display Content-Length"), flagSet.BoolVarP(&options.OutputServerHeader,"web-server","server", false, "Display Server header"), flagSet.BoolVarP(&options.OutputContentType,"content-type", "ct", false, "Display Content-Type header"), From 87c6ab8194be970a3c080420c6f8a4b74ea78f58 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 11 Oct 2021 05:23:26 +0530 Subject: [PATCH 037/355] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 0cdcb49..ecff256 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.3 + /_/ v1.1.4-dev ` // Version is the current version of httpx -const Version = `v1.1.3` +const Version = `v1.1.4-dev` // showBanner is used to show the banner to the user func showBanner() { From c410e6cb759dddcb3a88d04500c98ebe59c35ef6 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 11 Oct 2021 05:25:42 +0530 Subject: [PATCH 038/355] goreleaser update --- .goreleaser.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.goreleaser.yml b/.goreleaser.yml index ea6706a..88ba796 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -20,6 +20,8 @@ builds: goarch: '386' - goos: windows goarch: 'arm' + - goos: windows + goarch: 'arm64' binary: '{{ .ProjectName }}' main: cmd/httpx/httpx.go From b1e85238c62fdc90dfad766663d547eade79692c Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 11 Oct 2021 05:29:38 +0530 Subject: [PATCH 039/355] readme update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f2eb44..d56f5f4 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ INPUT: PROBES: -sc, -status-code Display Status Code - -tech, -tech-detect Display wappalyzer based technology detection + -td, -tech-detect Display wappalyzer based technology detection -cl, -content-length Display Content-Length -server, -web-server Display Server header -ct, -content-type Display Content-Type header From 0a26550cd200388081823c0a1268ab7c539aef0f Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 11 Oct 2021 05:36:29 +0530 Subject: [PATCH 040/355] readme updates --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d56f5f4..dd607f1 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ INPUT: PROBES: -sc, -status-code Display Status Code - -td, -tech-detect Display wappalyzer based technology detection + -td, -tech-detect Display wappalyzer based technology detection -cl, -content-length Display Content-Length -server, -web-server Display Server header -ct, -content-type Display Content-Type header @@ -156,12 +156,8 @@ CONFIGURATIONS: -vhost-input Get a list of vhosts as input -x string Request methods to use, use 'all' to probe all HTTP methods -body string Post body to include in HTTP request - -OPTIMIZATIONS: - -retries int Number of retries - -timeout int Timeout in seconds (default 5) - -maxhr, -max-host-error int Max error count per host before skipping remaining path/s (default 30) - -ec, -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) + -s, -stream Stream mode - start elaborating input targets without sorting + -sd, -skip-dedupe Disable dedupe input items (only used with stream mode) DEBUG: -silent Silent mode @@ -169,6 +165,12 @@ DEBUG: -version Display version -debug Debug mode -stats Display scan statistic + +OPTIMIZATIONS: + -retries int Number of retries + -timeout int Timeout in seconds (default 5) + -maxhr, -max-host-error int Max error count per host before skipping remaining path/s (default 30) + -ec, -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) ``` # Running httpX From 4a898404451ce8f425c01c272eabfbb3f8e8ef71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 10:04:38 +0000 Subject: [PATCH 041/355] chore(deps): bump golang from 1.17.1-alpine to 1.17.2-alpine Bumps golang from 1.17.1-alpine to 1.17.2-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index be4b7f9..2b9e30c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.1-alpine AS builder +FROM golang:1.17.2-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From d12543c87ffca107957cf7db5c36361704dd39b6 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 12 Oct 2021 02:38:51 +0530 Subject: [PATCH 042/355] docker push workflow update --- .github/workflows/dockerhub-push.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 065d143..0fd9d87 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -11,26 +11,30 @@ jobs: docker: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v2 - - - name: Set up QEMU + + - name: Get Github tag + id: meta + run: | + echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/httpx/releases/latest" | jq -r .tag_name)" + + - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx + + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub + + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - - name: Build and push + + - name: Build and push uses: docker/build-push-action@v2 with: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true - tags: projectdiscovery/httpx:latest \ No newline at end of file + tags: projectdiscovery/httpx:latest,projectdiscovery/httpx:${{ steps.meta.outputs.tag }} \ No newline at end of file From 20d51a1f75263484043fcf2cb7250520fd81ebf4 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 12 Oct 2021 09:33:50 +0200 Subject: [PATCH 043/355] Fixing stream mode with path parameter --- cmd/integration-test/http.go | 29 +++++++++++++++++++++++++++++ runner/runner.go | 6 +++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 460b087..0df2a04 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -22,6 +22,7 @@ var httpTestcases = map[string]testutils.TestCase{ "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body "Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body + "Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path } type standardHttpGet struct { @@ -209,3 +210,31 @@ func (h *issue400) Execute() error { } return nil } + +type issue414 struct{} + +func (h *issue414) Execute() error { + var ts *httptest.Server + uripath := "/path" + router := httprouter.New() + router.POST(uripath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + data, _ := ioutil.ReadAll(r.Body) + fmt.Fprintf(w, "data received %s", data) + })) + ts = httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL, debug, fmt.Sprintf("-path '%s'", uripath)) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + expected := ts.URL + uripath + if !strings.EqualFold(results[0], expected) { + return errIncorrectResult(results[0], expected) + } + return nil +} diff --git a/runner/runner.go b/runner/runner.go index f1fe054..8ecaefb 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -256,14 +256,16 @@ func New(options *Options) (*Runner, error) { return runner, nil } -func (r *Runner) prepareInput() { +func (r *Runner) prepareInputPaths() { // Check if the user requested multiple paths if fileutil.FileExists(r.options.RequestURIs) { r.options.requestURIs = fileutilz.LoadFile(r.options.RequestURIs) } else if r.options.RequestURIs != "" { r.options.requestURIs = strings.Split(r.options.RequestURIs, ",") } +} +func (r *Runner) prepareInput() { // check if file has been provided var numHosts int if fileutil.FileExists(r.options.InputFile) { @@ -488,6 +490,8 @@ func (r *Runner) RunEnumeration() { } } + r.prepareInputPaths() + var streamChan chan string if r.options.Stream { var err error From 9b53795521c5e65c1574721f81d587ddcc9d227e Mon Sep 17 00:00:00 2001 From: z0ne Date: Fri, 15 Oct 2021 15:31:08 +0800 Subject: [PATCH 044/355] Fixing follow-host-redirects option when customizing the host request header --- common/httpx/httpx.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 52491db..307fb1b 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -75,7 +75,10 @@ func New(options *Options) (*HTTPX, error) { redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { // Check if we get a redirect to a different host var newHost = redirectedRequest.URL.Host - var oldHost = previousRequests[0].URL.Host + var oldHost = previousRequests[0].Host + if oldHost == "" { + oldHost = previousRequests[0].URL.Host + } if newHost != oldHost { // Tell the http client to not follow redirect return http.ErrUseLastResponse From 1b89056d8a677b9c664f6a8c5694fa8f4e9237c3 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 16 Oct 2021 14:37:54 +0200 Subject: [PATCH 045/355] Adding max recursion level for self-dos on recursive load function --- common/customlist/customlist.go | 4 +++- common/fileutil/fileutil.go | 25 ++++++------------------- go.mod | 2 +- go.sum | 4 ++++ 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/common/customlist/customlist.go b/common/customlist/customlist.go index 4bf05f3..0969cf9 100644 --- a/common/customlist/customlist.go +++ b/common/customlist/customlist.go @@ -2,6 +2,8 @@ package customlist import "github.com/projectdiscovery/httpx/common/fileutil" +const maxRecursion = 10 + // CustomList for fastdialer type CustomList []string @@ -12,7 +14,7 @@ func (c *CustomList) String() string { // Set a new global header func (c *CustomList) Set(value string) error { - values := fileutil.LoadCidrsFromSliceOrFile(value, ",") + values := fileutil.LoadCidrsFromSliceOrFileWithMaxRecursion(value, ",", maxRecursion) *c = append(*c, values...) return nil } diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go index df5ce28..8fe5c6e 100644 --- a/common/fileutil/fileutil.go +++ b/common/fileutil/fileutil.go @@ -13,21 +13,6 @@ import ( "github.com/projectdiscovery/httpx/common/stringz" ) -// FileExists checks if a file exists and is not a directory -func FileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) || err != nil || info == nil { - return false - } - return !info.IsDir() -} - -// FolderExists checks if a folder exists -func FolderExists(folderpath string) bool { - _, err := os.Stat(folderpath) - return !os.IsNotExist(err) -} - // HasStdin determines if the user has piped input func HasStdin() bool { stat, err := os.Stdin.Stat() @@ -76,20 +61,22 @@ func FileNameIsGlob(pattern string) bool { return err == nil } -func LoadCidrsFromSliceOrFile(option string, splitchar string) (networkList []string) { +func LoadCidrsFromSliceOrFileWithMaxRecursion(option string, splitchar string, maxRecursion int) (networkList []string) { + if maxRecursion < 0 { + return + } items := stringz.SplitByCharAndTrimSpace(option, splitchar) for _, item := range items { - // ip if net.ParseIP(item) != nil { networkList = append(networkList, item) } else if _, _, err := net.ParseCIDR(item); err == nil { networkList = append(networkList, item) } else if fileutil.FileExists(item) { if filedata, err := ioutil.ReadFile(item); err == nil && len(filedata) > 0 { - networkList = append(networkList, LoadCidrsFromSliceOrFile(string(filedata), "\n")...) + networkList = append(networkList, LoadCidrsFromSliceOrFileWithMaxRecursion(string(filedata), "\n", maxRecursion-1)...) } } } - return networkList + return } diff --git a/go.mod b/go.mod index 3be225c..1ecdf09 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c + github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 @@ -62,7 +63,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect - github.com/projectdiscovery/goflags v0.0.7 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/syndtr/goleveldb v1.0.0 // indirect diff --git a/go.sum b/go.sum index 19a5754..bd15143 100644 --- a/go.sum +++ b/go.sum @@ -94,9 +94,11 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= @@ -187,6 +189,7 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= +github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= @@ -314,6 +317,7 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= From 3407748c222c35cf2dfe97201bc885caeb664396 Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 20 Oct 2021 04:15:46 +0530 Subject: [PATCH 046/355] Issue template update --- .github/ISSUE_TEMPLATE/config.yml | 14 ++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 22 ++++++++----- .github/ISSUE_TEMPLATE/issue-report.md | 40 ++++++++++++++--------- 3 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..4c851c7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +blank_issues_enabled: false + +contact_links: + - name: Ask an question / advise on using httpx + url: https://github.com/projectdiscovery/httpx/discussions/categories/q-a + about: Ask a question or request support for using httpx + + - name: Share idea / feature to discuss for httpx + url: https://github.com/projectdiscovery/httpx/discussions/categories/ideas + about: Share idea / feature to discuss for httpx + + - name: Connect with PD Team (Discord) + url: https://discord.gg/projectdiscovery + about: Connect with PD Team for direct communication \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index f3bde24..2b91595 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,14 +1,18 @@ --- name: Feature request -about: Suggest an idea for this project -title: "" -labels: '' -assignees: '' - +about: Request feature to implement in this project +labels: 'Type: Enhancement' --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + + +### Please describe your feature request: + -**Describe the solution you'd like** -A clear and concise description of what you want to happen. +### Describe the use case of this feature: + diff --git a/.github/ISSUE_TEMPLATE/issue-report.md b/.github/ISSUE_TEMPLATE/issue-report.md index b72b468..cbb53ca 100644 --- a/.github/ISSUE_TEMPLATE/issue-report.md +++ b/.github/ISSUE_TEMPLATE/issue-report.md @@ -1,26 +1,36 @@ --- name: Issue report -about: Create a report to help us improve -title: "" -labels: 'Type:+Bug' -assignees: '' +about: Create a report to help us to improve the project +labels: 'Type: Bug' --- -**Describe the bug** -A clear and concise description of what the bug is. + -**Environment details** -Please share the below details to help us quickly validate/replicate the issue. + -- `httpx -version` -- `go version` -- `uname -a` +### httpx version: + + + +### Current Behavior: + -**Error details** -Please copy-paste the error trace or details from terminal to help us quickly validate/replicate the issue. +### Expected Behavior: + -```bash +### Steps To Reproduce: + -``` \ No newline at end of file + +### Anything else: + \ No newline at end of file From 3e0a304fb9a565789f83147d8c0d01a7e27880fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Oct 2021 09:20:25 +0000 Subject: [PATCH 047/355] chore(deps): bump github.com/microcosm-cc/bluemonday Bumps [github.com/microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday) from 1.0.15 to 1.0.16. - [Release notes](https://github.com/microcosm-cc/bluemonday/releases) - [Commits](https://github.com/microcosm-cc/bluemonday/compare/v1.0.15...v1.0.16) --- updated-dependencies: - dependency-name: github.com/microcosm-cc/bluemonday dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3be225c..1592c58 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/microcosm-cc/bluemonday v1.0.15 + github.com/microcosm-cc/bluemonday v1.0.16 github.com/miekg/dns v1.1.43 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.2 @@ -22,6 +22,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c + github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 @@ -62,7 +63,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect - github.com/projectdiscovery/goflags v0.0.7 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/syndtr/goleveldb v1.0.0 // indirect diff --git a/go.sum b/go.sum index 19a5754..1502ea4 100644 --- a/go.sum +++ b/go.sum @@ -12,7 +12,6 @@ github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YT github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -94,16 +93,18 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= -github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= +github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc= +github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= @@ -187,6 +188,7 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= +github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= @@ -314,6 +316,7 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= From 13ad1e6a46e98df4ab267e7f413fc595954706a2 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 23 Oct 2021 13:59:51 +0300 Subject: [PATCH 048/355] Add request dump for debug flag --- runner/options.go | 4 ++++ runner/runner.go | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/runner/options.go b/runner/options.go index d88b002..d1881e0 100644 --- a/runner/options.go +++ b/runner/options.go @@ -167,6 +167,8 @@ type Options struct { OutputCName bool Unsafe bool Debug bool + DebugRequests bool + DebugResponse bool Pipeline bool HTTP2Probe bool OutputCDN bool @@ -295,6 +297,8 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose mode"), flagSet.BoolVar(&options.Version, "version", false, "Display version"), flagSet.BoolVar(&options.Debug, "debug", false, "Debug mode"), + flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "Show all sent requests"), + flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "Show all received responses"), flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Display scan statistic"), ) diff --git a/runner/runner.go b/runner/runner.go index 8ecaefb..1619006 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -846,6 +846,15 @@ retry: } fullURL = parsedURL.String() + if r.options.Debug || r.options.DebugRequests { + gologger.Info().Msgf("Dumped HTTP request for %s\n\n", fullURL) + gologger.Print().Msgf("%s", string(requestDump)) + } + if r.options.Debug || r.options.DebugResponse { + gologger.Info().Msgf("Dumped HTTP response for %s\n\n", fullURL) + gologger.Print().Msgf("%s", string(resp.Raw)) + } + builder := &strings.Builder{} builder.WriteString(stringz.RemoveURLDefaultPort(fullURL)) From 7158b3b0079066e65b1da3a3f23d8c916666f53c Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Sat, 23 Oct 2021 14:44:33 +0300 Subject: [PATCH 049/355] Update README with debug flags --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dd607f1..7b98b05 100644 --- a/README.md +++ b/README.md @@ -160,11 +160,13 @@ CONFIGURATIONS: -sd, -skip-dedupe Disable dedupe input items (only used with stream mode) DEBUG: - -silent Silent mode - -verbose Verbose mode - -version Display version - -debug Debug mode - -stats Display scan statistic + -silent Silent mode + -verbose Verbose mode + -version Display version + -debug Debug mode + -debug-req Show all sent requests + -debug-resp Show all received responses + -stats Display scan statistic OPTIMIZATIONS: -retries int Number of retries From d734fd2dbc9329458cd04dd2a40cf308501d6784 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Nov 2021 10:04:49 +0000 Subject: [PATCH 050/355] chore(deps): bump golang from 1.17.2-alpine to 1.17.3-alpine Bumps golang from 1.17.2-alpine to 1.17.3-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2b9e30c..d6b03a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.2-alpine AS builder +FROM golang:1.17.3-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 4d4db87346bb6452b1f989d6f441e3bd5b1ea065 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 11 Nov 2021 20:51:31 -0600 Subject: [PATCH 051/355] functional test --- cmd/functional-test/main.go | 83 +++++++++++++++++++ cmd/functional-test/run.sh | 13 +++ cmd/functional-test/test-data/raw-request.txt | 1 + cmd/functional-test/test-data/request.txt | 1 + cmd/functional-test/testcases.txt | 79 ++++++++++++++++++ internal/testutils/integration.go | 26 ++++++ 6 files changed, 203 insertions(+) create mode 100644 cmd/functional-test/main.go create mode 100755 cmd/functional-test/run.sh create mode 100644 cmd/functional-test/test-data/raw-request.txt create mode 100644 cmd/functional-test/test-data/request.txt create mode 100644 cmd/functional-test/testcases.txt diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go new file mode 100644 index 0000000..52a3db3 --- /dev/null +++ b/cmd/functional-test/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + + "github.com/projectdiscovery/httpx/internal/testutils" +) + +var ( + debug = os.Getenv("DEBUG") == "true" + success = aurora.Green("[✓]").String() + failed = aurora.Red("[✘]").String() + errored = false + + mainHttpxBinary = flag.String("main", "", "Main Branch Httpx Binary") + devHttpxBinary = flag.String("dev", "", "Dev Branch Httpx Binary") + testcases = flag.String("testcases", "", "Test cases file for Httpx functional tests") +) + +func main() { + flag.Parse() + + if err := runFunctionalTests(); err != nil { + log.Fatalf("Could not run functional tests: %s\n", err) + } + if errored { + os.Exit(1) + } +} + +func runFunctionalTests() error { + file, err := os.Open(*testcases) + if err != nil { + return errors.Wrap(err, "could not open test cases") + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if text == "" { + continue + } + if err := runIndividualTestCase(text); err != nil { + errored = true + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err) + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, text) + } + } + return nil +} + +func runIndividualTestCase(testcase string) error { + parts := strings.Fields(testcase) + + var finalArgs []string + var target string + if len(parts) > 1 { + finalArgs = parts[2:] + target = parts[0] + } + mainOutput, err := testutils.RunHttpxBinaryAndGetResults(target, *mainHttpxBinary, debug, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run httpx main test") + } + devOutput, err := testutils.RunHttpxBinaryAndGetResults(target, *devHttpxBinary, debug, finalArgs) + if err != nil { + return errors.Wrap(err, "could not run httpx dev test") + } + if len(mainOutput) == len(devOutput) { + return nil + } + return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput) +} diff --git a/cmd/functional-test/run.sh b/cmd/functional-test/run.sh new file mode 100755 index 0000000..be5e8c7 --- /dev/null +++ b/cmd/functional-test/run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +echo 'Building functional-test binary' +go build + +echo 'Building HTTPX binary from current branch' +go build -o httpx_dev ../httpx + +echo 'Installing latest release of HTTPX' +GO111MODULE=on go build -v github.com/projectdiscovery/httpx/cmd/httpx + +echo 'Starting HTTPX functional test' +./functional-test -main ./httpx -dev ./httpx_dev -testcases testcases.txt diff --git a/cmd/functional-test/test-data/raw-request.txt b/cmd/functional-test/test-data/raw-request.txt new file mode 100644 index 0000000..94b072c --- /dev/null +++ b/cmd/functional-test/test-data/raw-request.txt @@ -0,0 +1 @@ +GET /search?q=test HTTP/2 diff --git a/cmd/functional-test/test-data/request.txt b/cmd/functional-test/test-data/request.txt new file mode 100644 index 0000000..284d3f2 --- /dev/null +++ b/cmd/functional-test/test-data/request.txt @@ -0,0 +1 @@ +https://www.example.com diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt new file mode 100644 index 0000000..90c4f69 --- /dev/null +++ b/cmd/functional-test/testcases.txt @@ -0,0 +1,79 @@ +www.example.com {{binary}} +www.example.com {{binary}} -l test-data/request.txt +www.example.com {{binary}} -request test-data/raw-request.txt +www.example.com {{binary}} -status-code +www.example.com {{binary}} -tech-detect +www.example.com {{binary}} -content-length +www.example.com {{binary}} -web-server +www.example.com {{binary}} -content-type +www.example.com {{binary}} -response-time +www.example.com {{binary}} -title +www.example.com {{binary}} -location +www.example.com {{binary}} -method +www.example.com {{binary}} -websocket +www.example.com {{binary}} -ip +www.example.com {{binary}} -cname +www.example.com {{binary}} -cdn +www.example.com {{binary}} -probe +www.example.com {{binary}} -no-fallback +www.example.com {{binary}} -match-code 200, -mc 200 +www.example.com {{binary}} -match-length 100, -ml 100 +www.example.com {{binary}} -match-string xyz, -ms xyz +www.example.com {{binary}} -match-regex bbb, -mr bbb +www.example.com {{binary}} -extract-regex aaa, -er aaa +www.example.com {{binary}} -filter-code 401, -fc 401 +www.example.com {{binary}} -filter-length 23, -fl 23 +www.example.com {{binary}} -filter-string +www.example.com {{binary}} -filter-regex +www.example.com {{binary}} -threads 50 +www.example.com {{binary}} -rate-limit 5 +www.example.com {{binary}} -tls-grab +www.example.com {{binary}} -tls-probe +www.example.com {{binary}} -csp-probe +www.example.com {{binary}} -pipeline +www.example.com {{binary}} -http2 +www.example.com {{binary}} -vhost +www.example.com {{binary}} -ports +www.example.com {{binary}} -path +www.example.com {{binary}} -paths +www.example.com {{binary}} -output test-data/request.txt +www.example.com {{binary}} -store-response +www.example.com {{binary}} -store-response-dir test-data/request.txt +www.example.com {{binary}} -json +www.example.com {{binary}} -include-response -json +www.example.com {{binary}} -include-chain +www.example.com {{binary}} -store-chain +www.example.com {{binary}} -csv +www.example.com {{binary}} -response-size-to-save 1028 +www.example.com {{binary}} -response-size-to-read 1028 +www.example.com {{binary}} -allow 127.0.0.1:8080 +www.example.com {{binary}} -deny 127.0.0.1:6089 +www.example.com {{binary}} -random-agent +www.example.com {{binary}} -header +www.example.com {{binary}} -proxy www.example.com +www.example.com {{binary}} -unsafe +www.example.com {{binary}} -resume +www.example.com {{binary}} -no-color +www.example.com {{binary}} -no-fallback-scheme +www.example.com {{binary}} -follow-redirects +www.example.com {{binary}} -follow-host-redirects +www.example.com {{binary}} -max-redirects 10 +www.example.com {{binary}} -vhost-input +www.example.com {{binary}} -x +www.example.com {{binary}} -body +www.example.com {{binary}} -stream +www.example.com {{binary}} -skip-dedupe -stream +www.example.com {{binary}} -silent +www.example.com {{binary}} -verbose +www.example.com {{binary}} -version +www.example.com {{binary}} -debug +www.example.com {{binary}} -debug-req +www.example.com {{binary}} -debug-resp +www.example.com {{binary}} -stats +www.example.com {{binary}} -retries 0 +www.example.com {{binary}} -timeout 10 +www.example.com {{binary}} -max-host-error 30 +www.example.com {{binary}} -exclude-cdn + + + diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go index 6f7b28b..d058952 100644 --- a/internal/testutils/integration.go +++ b/internal/testutils/integration.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "strings" + "fmt" ) // RunNucleiAndGetResults returns a list of results for a template @@ -33,6 +34,31 @@ func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, e } return parts, nil } +func RunHttpxBinaryAndGetResults(target string, httpxBinary string, debug bool, args []string) ([]string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := fmt.Sprintf(`echo %s | %s `, target, httpxBinary) + cmdLine += strings.Join(args, " ") + if debug { + cmdLine += " -debug" + cmd.Stderr = os.Stderr + } else { + cmdLine += " -silent" + } + + cmd.Args = append(cmd.Args, cmdLine) + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts,nil +} // TestCase is a single integration test case type TestCase interface { From 8eca9609e1454439cef7601a2080804b27ab9215 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 10:11:27 +0000 Subject: [PATCH 052/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.13 to 0.0.16. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.13...v0.0.16) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1592c58..a2f1050 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.13 + github.com/projectdiscovery/wappalyzergo v0.0.16 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1502ea4..7f55db3 100644 --- a/go.sum +++ b/go.sum @@ -193,8 +193,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.13 h1:00Kc1QE5YGqd8u6jd3E8SZQk3eQhjGyme5g2eAJtdCY= -github.com/projectdiscovery/wappalyzergo v0.0.13/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.16 h1:75DMqE6/JONzNMBH2uFM0/9Fy/iPiOGwu16nnyE/fXw= +github.com/projectdiscovery/wappalyzergo v0.0.16/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= From 77ec79caafb2f5221756eba577421e777044bfcf Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 16 Nov 2021 18:23:43 -0600 Subject: [PATCH 053/355] github action for functional test --- .github/workflows/functional-test.yml | 25 ++++++ .gitignore | 7 +- cmd/functional-test/testcases.txt | 120 ++++++++++---------------- 3 files changed, 76 insertions(+), 76 deletions(-) create mode 100644 .github/workflows/functional-test.yml diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml new file mode 100644 index 0000000..27e9a63 --- /dev/null +++ b/.github/workflows/functional-test.yml @@ -0,0 +1,25 @@ +name: 🧪 Functional Test + on: + push: + pull_request: + workflow_dispatch: + + + jobs: + functional: + name: Functional Test + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Check out code + uses: actions/checkout@v2 + + - name: Functional Tests + run: | + chmod +x run.sh + bash run.sh + working-directory: cmd/functional-test diff --git a/.gitignore b/.gitignore index 3b6ed0b..2390b4e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,9 @@ cmd/httpx/httpx integration_tests/httpx -integration_tests/integration-test \ No newline at end of file +integration_tests/integration-test +cmd/functional-test/httpx_dev +cmd/functional-test/functional-test +cmd/functional-test/httpx +cmd/functional-test/*.cfg + diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index 90c4f69..d794dcf 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -1,79 +1,49 @@ -www.example.com {{binary}} -www.example.com {{binary}} -l test-data/request.txt -www.example.com {{binary}} -request test-data/raw-request.txt -www.example.com {{binary}} -status-code -www.example.com {{binary}} -tech-detect -www.example.com {{binary}} -content-length -www.example.com {{binary}} -web-server -www.example.com {{binary}} -content-type -www.example.com {{binary}} -response-time -www.example.com {{binary}} -title -www.example.com {{binary}} -location -www.example.com {{binary}} -method -www.example.com {{binary}} -websocket -www.example.com {{binary}} -ip -www.example.com {{binary}} -cname -www.example.com {{binary}} -cdn -www.example.com {{binary}} -probe -www.example.com {{binary}} -no-fallback -www.example.com {{binary}} -match-code 200, -mc 200 -www.example.com {{binary}} -match-length 100, -ml 100 -www.example.com {{binary}} -match-string xyz, -ms xyz -www.example.com {{binary}} -match-regex bbb, -mr bbb -www.example.com {{binary}} -extract-regex aaa, -er aaa -www.example.com {{binary}} -filter-code 401, -fc 401 -www.example.com {{binary}} -filter-length 23, -fl 23 -www.example.com {{binary}} -filter-string -www.example.com {{binary}} -filter-regex -www.example.com {{binary}} -threads 50 -www.example.com {{binary}} -rate-limit 5 -www.example.com {{binary}} -tls-grab -www.example.com {{binary}} -tls-probe -www.example.com {{binary}} -csp-probe -www.example.com {{binary}} -pipeline -www.example.com {{binary}} -http2 -www.example.com {{binary}} -vhost -www.example.com {{binary}} -ports -www.example.com {{binary}} -path -www.example.com {{binary}} -paths -www.example.com {{binary}} -output test-data/request.txt -www.example.com {{binary}} -store-response -www.example.com {{binary}} -store-response-dir test-data/request.txt -www.example.com {{binary}} -json -www.example.com {{binary}} -include-response -json -www.example.com {{binary}} -include-chain -www.example.com {{binary}} -store-chain -www.example.com {{binary}} -csv -www.example.com {{binary}} -response-size-to-save 1028 -www.example.com {{binary}} -response-size-to-read 1028 -www.example.com {{binary}} -allow 127.0.0.1:8080 -www.example.com {{binary}} -deny 127.0.0.1:6089 +www.example.com {{binary}} -silent +www.example.com {{binary}} -silent -l test-data/request.txt +www.example.com {{binary}} -silent -request test-data/raw-request.txt +www.example.com {{binary}} -silent -response-time +www.example.com {{binary}} -silent -title +www.example.com {{binary}} -silent -location +www.example.com {{binary}} -silent -cname +www.example.com {{binary}} -silent -cdn +www.example.com {{binary}} -silent -probe +www.example.com {{binary}} -silent -no-fallback +www.example.com {{binary}} -silent -match-code 200, -mc 200 +www.example.com {{binary}} -silent -match-length 100, -ml 100 +www.example.com {{binary}} -silent -match-string xyz, -ms xyz +www.example.com {{binary}} -silent -match-regex bbb, -mr bbb +www.example.com {{binary}} -silent -filter-code 401, -fc 401 +www.example.com {{binary}} -silent -filter-length 23, -fl 23 +www.example.com {{binary}} -silent -filter-string +www.example.com {{binary}} -silent -threads 50 +www.example.com {{binary}} -silent -rate-limit 5 +www.example.com {{binary}} -silent -tls-grab +www.example.com {{binary}} -silent -pipeline +www.example.com {{binary}} -silent -http2 +www.example.com {{binary}} -silent -ports +www.example.com {{binary}} -silent -paths +www.example.com {{binary}} -silent -output test-data/request.txt +www.example.com {{binary}} -silent -store-response +www.example.com {{binary}} -silent -include-response -json +www.example.com {{binary}} -silent -include-chain +www.example.com {{binary}} -silent -csv +www.example.com {{binary}} -silent -response-size-to-save 1028 +www.example.com {{binary}} -silent -response-size-to-read 1028 +www.example.com {{binary}} -silent -allow 127.0.0.1:8080 +www.example.com {{binary}} -silent -deny 127.0.0.1:6089 www.example.com {{binary}} -random-agent -www.example.com {{binary}} -header -www.example.com {{binary}} -proxy www.example.com -www.example.com {{binary}} -unsafe -www.example.com {{binary}} -resume -www.example.com {{binary}} -no-color -www.example.com {{binary}} -no-fallback-scheme -www.example.com {{binary}} -follow-redirects -www.example.com {{binary}} -follow-host-redirects -www.example.com {{binary}} -max-redirects 10 -www.example.com {{binary}} -vhost-input -www.example.com {{binary}} -x -www.example.com {{binary}} -body -www.example.com {{binary}} -stream -www.example.com {{binary}} -skip-dedupe -stream -www.example.com {{binary}} -silent -www.example.com {{binary}} -verbose -www.example.com {{binary}} -version -www.example.com {{binary}} -debug -www.example.com {{binary}} -debug-req -www.example.com {{binary}} -debug-resp -www.example.com {{binary}} -stats -www.example.com {{binary}} -retries 0 -www.example.com {{binary}} -timeout 10 -www.example.com {{binary}} -max-host-error 30 -www.example.com {{binary}} -exclude-cdn +www.example.com {{binary}} -silent -proxy www.example.com +www.example.com {{binary}} -silent -unsafe +www.example.com {{binary}} -silent -resume +www.example.com {{binary}} -silent -max-redirects 10 +www.example.com {{binary}} -silent -vhost-input +www.example.com {{binary}} -silent -x +www.example.com {{binary}} -silent -body +www.example.com {{binary}} -silent -skip-dedupe -stream +www.example.com {{binary}} -silent -retries 10 +www.example.com {{binary}} -silent -timeout 10 +www.example.com {{binary}} -silent -max-host-error 30 +www.example.com {{binary}} -silent -exclude-cdn From b46600a3f2d45c0ffccc74080bb28417b35de440 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 16 Nov 2021 18:32:05 -0600 Subject: [PATCH 054/355] github action for functional test --- .github/workflows/functional-test.yml | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 27e9a63..027f3c7 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -1,25 +1,25 @@ name: 🧪 Functional Test - on: - push: - pull_request: - workflow_dispatch: +on: + push: + pull_request: + workflow_dispatch: - jobs: - functional: - name: Functional Test - runs-on: ubuntu-latest - steps: - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.17 +jobs: + functional: + name: Functional Test + runs-on: ubuntu-latest + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 - - name: Check out code - uses: actions/checkout@v2 + - name: Check out code + uses: actions/checkout@v2 - - name: Functional Tests - run: | - chmod +x run.sh - bash run.sh - working-directory: cmd/functional-test + - name: Functional Tests + run: | + chmod +x run.sh + bash run.sh + working-directory: cmd/functional-test From a4512bcca6fbef195dd6e8f66eda35f9d9fa864f Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 17 Nov 2021 10:24:14 -0600 Subject: [PATCH 055/355] removed commas and added missing arguments to flags --- cmd/functional-test/testcases.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index d794dcf..a42ef8a 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -8,20 +8,20 @@ www.example.com {{binary}} -silent -cname www.example.com {{binary}} -silent -cdn www.example.com {{binary}} -silent -probe www.example.com {{binary}} -silent -no-fallback -www.example.com {{binary}} -silent -match-code 200, -mc 200 -www.example.com {{binary}} -silent -match-length 100, -ml 100 -www.example.com {{binary}} -silent -match-string xyz, -ms xyz -www.example.com {{binary}} -silent -match-regex bbb, -mr bbb -www.example.com {{binary}} -silent -filter-code 401, -fc 401 -www.example.com {{binary}} -silent -filter-length 23, -fl 23 -www.example.com {{binary}} -silent -filter-string +www.example.com {{binary}} -silent -match-code 200 -mc 200 +www.example.com {{binary}} -silent -match-length 100 -ml 100 +www.example.com {{binary}} -silent -match-string xyz -ms xyz +www.example.com {{binary}} -silent -match-regex bbb -mr bbb +www.example.com {{binary}} -silent -filter-code 401 -fc 401 +www.example.com {{binary}} -silent -filter-length 23 -fl 23 +www.example.com {{binary}} -silent -filter-string example www.example.com {{binary}} -silent -threads 50 www.example.com {{binary}} -silent -rate-limit 5 www.example.com {{binary}} -silent -tls-grab www.example.com {{binary}} -silent -pipeline www.example.com {{binary}} -silent -http2 -www.example.com {{binary}} -silent -ports -www.example.com {{binary}} -silent -paths +www.example.com {{binary}} -silent -ports 8080,6089 +www.example.com {{binary}} -silent -paths test-data/request.txt www.example.com {{binary}} -silent -output test-data/request.txt www.example.com {{binary}} -silent -store-response www.example.com {{binary}} -silent -include-response -json @@ -37,8 +37,8 @@ www.example.com {{binary}} -silent -unsafe www.example.com {{binary}} -silent -resume www.example.com {{binary}} -silent -max-redirects 10 www.example.com {{binary}} -silent -vhost-input -www.example.com {{binary}} -silent -x -www.example.com {{binary}} -silent -body +www.example.com {{binary}} -silent -x all +www.example.com {{binary}} -silent -body 'a=b' www.example.com {{binary}} -silent -skip-dedupe -stream www.example.com {{binary}} -silent -retries 10 www.example.com {{binary}} -silent -timeout 10 From 56535375a5074b42186c903b9cb0bffd4104d817 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 18 Nov 2021 19:12:17 +0530 Subject: [PATCH 056/355] testcases update --- cmd/functional-test/testcases.txt | 43 ++++--------------------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index a42ef8a..e95bdc3 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -1,49 +1,16 @@ www.example.com {{binary}} -silent www.example.com {{binary}} -silent -l test-data/request.txt www.example.com {{binary}} -silent -request test-data/raw-request.txt -www.example.com {{binary}} -silent -response-time www.example.com {{binary}} -silent -title -www.example.com {{binary}} -silent -location -www.example.com {{binary}} -silent -cname -www.example.com {{binary}} -silent -cdn +www.example.com {{binary}} -silent -sc +www.example.com {{binary}} -silent -td www.example.com {{binary}} -silent -probe www.example.com {{binary}} -silent -no-fallback -www.example.com {{binary}} -silent -match-code 200 -mc 200 -www.example.com {{binary}} -silent -match-length 100 -ml 100 -www.example.com {{binary}} -silent -match-string xyz -ms xyz -www.example.com {{binary}} -silent -match-regex bbb -mr bbb -www.example.com {{binary}} -silent -filter-code 401 -fc 401 -www.example.com {{binary}} -silent -filter-length 23 -fl 23 -www.example.com {{binary}} -silent -filter-string example -www.example.com {{binary}} -silent -threads 50 -www.example.com {{binary}} -silent -rate-limit 5 +www.example.com {{binary}} -silent -cl +www.example.com {{binary}} -silent -server +www.example.com {{binary}} -silent -ip www.example.com {{binary}} -silent -tls-grab -www.example.com {{binary}} -silent -pipeline -www.example.com {{binary}} -silent -http2 -www.example.com {{binary}} -silent -ports 8080,6089 -www.example.com {{binary}} -silent -paths test-data/request.txt -www.example.com {{binary}} -silent -output test-data/request.txt -www.example.com {{binary}} -silent -store-response -www.example.com {{binary}} -silent -include-response -json -www.example.com {{binary}} -silent -include-chain -www.example.com {{binary}} -silent -csv -www.example.com {{binary}} -silent -response-size-to-save 1028 -www.example.com {{binary}} -silent -response-size-to-read 1028 -www.example.com {{binary}} -silent -allow 127.0.0.1:8080 -www.example.com {{binary}} -silent -deny 127.0.0.1:6089 -www.example.com {{binary}} -random-agent -www.example.com {{binary}} -silent -proxy www.example.com www.example.com {{binary}} -silent -unsafe -www.example.com {{binary}} -silent -resume -www.example.com {{binary}} -silent -max-redirects 10 -www.example.com {{binary}} -silent -vhost-input www.example.com {{binary}} -silent -x all www.example.com {{binary}} -silent -body 'a=b' -www.example.com {{binary}} -silent -skip-dedupe -stream -www.example.com {{binary}} -silent -retries 10 -www.example.com {{binary}} -silent -timeout 10 -www.example.com {{binary}} -silent -max-host-error 30 www.example.com {{binary}} -silent -exclude-cdn - - - From 64734dd13a415d2abf2346867b97e1211ac9d5dc Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Mon, 22 Nov 2021 09:40:32 +0300 Subject: [PATCH 057/355] Fix npe on debug output --- runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 1619006..f743838 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -850,7 +850,7 @@ retry: gologger.Info().Msgf("Dumped HTTP request for %s\n\n", fullURL) gologger.Print().Msgf("%s", string(requestDump)) } - if r.options.Debug || r.options.DebugResponse { + if (r.options.Debug || r.options.DebugResponse) && resp != nil { gologger.Info().Msgf("Dumped HTTP response for %s\n\n", fullURL) gologger.Print().Msgf("%s", string(resp.Raw)) } From e9f847dc925d130630e0782a4f5feff9a6d1653a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Nov 2021 10:07:53 +0000 Subject: [PATCH 058/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.16 to 0.0.17. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.16...v0.0.17) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a2f1050..8b6617d 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.16 + github.com/projectdiscovery/wappalyzergo v0.0.17 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7f55db3..fd14eff 100644 --- a/go.sum +++ b/go.sum @@ -193,8 +193,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.16 h1:75DMqE6/JONzNMBH2uFM0/9Fy/iPiOGwu16nnyE/fXw= -github.com/projectdiscovery/wappalyzergo v0.0.16/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.17 h1:IodDbINx/UPXvv3Aibik9v1KxC3p+tPcPYMkTAtSnVU= +github.com/projectdiscovery/wappalyzergo v0.0.17/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= From e694badff2f677827d2a03a184ca98c88d536df7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 10:06:14 +0000 Subject: [PATCH 059/355] chore(deps): bump alpine from 3.14 to 3.15.0 Bumps alpine from 3.14 to 3.15.0. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d6b03a7..d16b9ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.3-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.14 +FROM alpine:3.15.0 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From edebd6a018cc97658f122d5e27a4e54b5cead067 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 10:10:14 +0000 Subject: [PATCH 060/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.17 to 0.0.18. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.17...v0.0.18) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8b6617d..0684e85 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.17 + github.com/projectdiscovery/wappalyzergo v0.0.18 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index fd14eff..bf1710b 100644 --- a/go.sum +++ b/go.sum @@ -193,8 +193,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.17 h1:IodDbINx/UPXvv3Aibik9v1KxC3p+tPcPYMkTAtSnVU= -github.com/projectdiscovery/wappalyzergo v0.0.17/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.18 h1:R2BcCFE777QEtON+mnKyVbD4qSIu35Ppe0UPR7kDw3s= +github.com/projectdiscovery/wappalyzergo v0.0.18/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= From 59ebd48e885a6367a0cd533203eeb6fd62d360c4 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 29 Nov 2021 13:15:59 -0600 Subject: [PATCH 061/355] bug-fix new line with title flag --- common/httpx/title.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/httpx/title.go b/common/httpx/title.go index 1edb534..ba1c7e3 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -34,6 +34,8 @@ func ExtractTitle(r *Response) (title string) { // remove unwanted chars title = strings.TrimSpace(strings.Trim(title, cutset)) + title = strings.ReplaceAll(title, "\n", "") + title = strings.ReplaceAll(title, "\r", "") // Non UTF-8 if contentTypes, ok := r.Headers["Content-Type"]; ok { From 69b0bb5fec6dbf2d97ee5be72419482ea4c4fd0f Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 29 Nov 2021 13:16:37 -0600 Subject: [PATCH 062/355] test case addition --- cmd/integration-test/http.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 0df2a04..95a458d 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -23,6 +23,7 @@ var httpTestcases = map[string]testutils.TestCase{ "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body "Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body "Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path + "Regression test for: https://github.com/projectdiscovery/httpx/issues/433": &issue433{}, // new line scanning with title flag } type standardHttpGet struct { @@ -238,3 +239,28 @@ func (h *issue414) Execute() error { } return nil } + +type issue433 struct{} + +func (h *issue433) Execute() error { + var ts *httptest.Server + router := httprouter.New() + uriPath := "/index" + router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + htmlResponse := "Project\n\r Discovery\n - Httpx>test data" + fmt.Fprint(w, htmlResponse) + })) + ts = httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunHttpxAndGetResults(fmt.Sprint(ts.URL, uriPath), debug, "-title", "-no-color") + if err != nil { + return err + } + if strings.Contains(results[0], "\n") { + return errIncorrectResultsCount(results) + } + if strings.Contains(results[0], "\r") { + return errIncorrectResultsCount(results) + } + return nil +} From 94815945281c1510c383f43194d5f7090a3233f7 Mon Sep 17 00:00:00 2001 From: Alexey Zhuchkov Date: Wed, 1 Dec 2021 10:10:53 +0300 Subject: [PATCH 063/355] Add probe-all-ips feature (#427) * Add probe-all-ips flag Co-authored-by: mzack Co-authored-by: sandeep --- common/httpx/httpx.go | 9 +++- go.mod | 22 +++------ go.sum | 102 +++++++++++------------------------------- runner/options.go | 76 ++++++++++++++++--------------- runner/runner.go | 67 ++++++++++++++++++++------- 5 files changed, 131 insertions(+), 145 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 307fb1b..b7c84d2 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/stringsutil" + "golang.org/x/net/context" "golang.org/x/net/http2" ) @@ -42,6 +43,7 @@ func New(options *Options) (*HTTPX, error) { fastdialerOpts.EnableFallback = true fastdialerOpts.Deny = options.Deny fastdialerOpts.Allow = options.Allow + fastdialerOpts.WithDialerHistory = true dialer, err := fastdialer.NewDialer(fastdialerOpts) if err != nil { return nil, fmt.Errorf("could not create resolver cache: %s", err) @@ -294,7 +296,12 @@ func (h *HTTPX) AddFilter(f Filter) { // NewRequest from url func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request, err error) { - req, err = retryablehttp.NewRequest(method, targetURL, nil) + return h.NewRequestWithContext(context.Background(), method, targetURL) +} + +// NewRequest from url +func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL string) (req *retryablehttp.Request, err error) { + req, err = retryablehttp.NewRequestWithContext(ctx, method, targetURL, nil) if err != nil { return } diff --git a/go.mod b/go.mod index 0684e85..93d9a11 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,6 @@ require ( github.com/akrylysov/pogreb v0.10.1 // indirect github.com/bluele/gcache v0.0.2 github.com/corpix/uarand v0.1.1 - github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 @@ -18,19 +16,19 @@ require ( github.com/projectdiscovery/cdncheck v0.0.2 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 + github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 - github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa + github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 - github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b + github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 github.com/projectdiscovery/mapcidr v0.0.8 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e - github.com/projectdiscovery/retryabledns v1.0.12 // indirect - github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc + github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a // indirect + github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 @@ -41,22 +39,16 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 + golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect golang.org/x/text v0.3.7 - google.golang.org/protobuf v1.27.1 // indirect ) require ( - github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/dgraph-io/badger v1.6.2 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/karrick/godirwalk v1.16.1 // indirect diff --git a/go.sum b/go.sum index bf1710b..36b9d10 100644 --- a/go.sum +++ b/go.sum @@ -1,59 +1,30 @@ -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= -github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v0.0.0-20210429001901-424d2337a529 h1:2voWjNECnrZRbfwXxHB1/j8wa6xdKn85B5NzgVL/pTU= -github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -61,7 +32,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -71,18 +41,15 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -92,9 +59,10 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -102,15 +70,12 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc= github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -131,8 +96,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -145,8 +109,8 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80 h1:Hyt1AcK5hAgE7LuaNyRD3mFc8zaS7LP4vTk6Yp83f8E= -github.com/projectdiscovery/fastdialer v0.0.13-0.20210815100514-360f851a5b80/go.mod h1:RkRbxqDCcCFhfNUbkzBIz/ieD4uda2JuUA4WJ+RLee0= +github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e h1:ORAzxUm2Dodjp2le/OqCD7S5KDqewV0F1REhabb2fD4= +github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21 h1:YtKqlzdhAfXbV6R7QrcoaP21+lpb6G++GEDqJbeEmQA= @@ -159,18 +123,17 @@ github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLY github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= -github.com/projectdiscovery/hmap v0.0.2-0.20210616215655-7b78e7f33d1f/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= -github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa h1:KeN6/bZOVxtS4XkgzRvYxpXWZSZt+AoGP5Myyr3/Duk= -github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa/go.mod h1:FH+MS/WNKTXJQtdRn+/Zg5WlKCiMN0Z1QUedUIuM5n8= +github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= +github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= -github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= -github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b h1:ajuou9cW3DGEtArQejqEyDMHcmhXEF7ASgsdVwBkvzs= -github.com/projectdiscovery/iputil v0.0.0-20210705072957-5a968407979b/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= +github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 h1:VZ9H51A7tI7R/9I5W5l960Nkq7eMJqGd3y1wsuwzdjE= +github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= +github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo2zlsL4L4E= github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= @@ -179,16 +142,16 @@ github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpG github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= -github.com/projectdiscovery/retryabledns v1.0.11/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= -github.com/projectdiscovery/retryabledns v1.0.12 h1:OzCsUaipN75OwjtH62FxBIhKye1NmnfG4DxtQclOtns= -github.com/projectdiscovery/retryabledns v1.0.12/go.mod h1:4sMC8HZyF01HXukRleSQYwz4870bwgb4+hTSXTMrkf4= +github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE= +github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= -github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc h1:769c7sQOl9BP8dhE8uv0mQX9WmcXo6Jzv//Nm+qAf50= -github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= +github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= +github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= @@ -197,24 +160,17 @@ github.com/projectdiscovery/wappalyzergo v0.0.18 h1:R2BcCFE777QEtON+mnKyVbD4qSIu github.com/projectdiscovery/wappalyzergo v0.0.18/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -223,8 +179,6 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -236,7 +190,6 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -254,21 +207,18 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -285,9 +235,8 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= +golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -302,7 +251,6 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -312,12 +260,12 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/runner/options.go b/runner/options.go index d1881e0..8bb3347 100644 --- a/runner/options.go +++ b/runner/options.go @@ -63,6 +63,7 @@ type scanOptions struct { extractRegex *regexp.Regexp ExcludeCDN bool HostMaxErrors int + ProbeAllIPS bool } func (s *scanOptions) Clone() *scanOptions { @@ -194,6 +195,7 @@ type Options struct { HostMaxErrors int Stream bool SkipDedupe bool + ProbeAllIPS bool } // ParseOptions parses the command line options for application @@ -204,17 +206,17 @@ func ParseOptions() *Options { flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library.`) createGroup(flagSet, "input", "Input", - flagSet.StringVarP(&options.InputFile,"list", "l", "", "Input file containing list of hosts to process"), + flagSet.StringVarP(&options.InputFile, "list", "l", "", "Input file containing list of hosts to process"), flagSet.StringVar(&options.InputRawRequest, "request", "", "File containing raw request"), ) createGroup(flagSet, "Probes", "Probes", - flagSet.BoolVarP(&options.StatusCode,"status-code", "sc", false, "Display Status Code"), - flagSet.BoolVarP(&options.TechDetect,"tech-detect", "td", false, "Display wappalyzer based technology detection"), - flagSet.BoolVarP(&options.ContentLength,"content-length", "cl", false, "Display Content-Length"), - flagSet.BoolVarP(&options.OutputServerHeader,"web-server","server", false, "Display Server header"), - flagSet.BoolVarP(&options.OutputContentType,"content-type", "ct", false, "Display Content-Type header"), - flagSet.BoolVarP(&options.OutputResponseTime,"response-time", "rt", false, "Display the response time"), + flagSet.BoolVarP(&options.StatusCode, "status-code", "sc", false, "Display Status Code"), + flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "Display wappalyzer based technology detection"), + flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "Display Content-Length"), + flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "Display Server header"), + flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "Display Content-Type header"), + flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "Display the response time"), flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"), flagSet.BoolVar(&options.Location, "location", false, "Display Location header"), flagSet.BoolVar(&options.OutputMethod, "method", false, "Display Request method"), @@ -223,27 +225,26 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"), flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), - flagSet.BoolVarP(&options.NoFallback,"no-fallback", "nf", false, "Display both protocol (HTTPS and HTTP)"), ) createGroup(flagSet, "matchers", "Matchers", - flagSet.StringVarP(&options.OutputMatchStatusCode,"match-code", "mc", "", "Match response with given status code (-mc 200,302)"), - flagSet.StringVarP(&options.OutputMatchContentLength,"match-length", "ml", "", "Match response with given content length (-ml 100,102)"), - flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms","", "Match response with given string"), - flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr","", "Match response with specific regex"), - flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er","", "Display response content with matched regex"), + flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "Match response with given status code (-mc 200,302)"), + flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"), + flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"), + flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"), + flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), ) createGroup(flagSet, "filters", "Filters", - flagSet.StringVarP(&options.OutputFilterStatusCode,"filter-code", "fc", "", "Filter response with given status code (-fc 403,401)"), - flagSet.StringVarP(&options.OutputFilterContentLength,"filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"), + flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "Filter response with given status code (-fc 403,401)"), + flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"), flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"), - flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe","", "Filter response with specific regex"), + flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", - flagSet.IntVarP(&options.Threads, "threads","t", 50, "Number of threads"), - flagSet.IntVarP(&options.RateLimit,"rate-limit","rl", 150, "Maximum requests to send per second"), + flagSet.IntVarP(&options.Threads, "threads", "t", 50, "Number of threads"), + flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second"), ) createGroup(flagSet, "Misc", "Miscellaneous", @@ -253,49 +254,46 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe"), flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe"), flagSet.BoolVar(&options.VHost, "vhost", false, "VHOST Probe"), - flagSet.VarP(&options.CustomPorts,"ports","p", "Port to scan (nmap syntax: eg 1,2-10,11)"), + flagSet.VarP(&options.CustomPorts, "ports", "p", "Port to scan (nmap syntax: eg 1,2-10,11)"), flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"), flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"), ) createGroup(flagSet, "output", "Output", - flagSet.StringVarP(&options.Output,"output", "o", "", "File to write output"), + flagSet.StringVarP(&options.Output, "output", "o", "", "File to write output"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "Store HTTP responses"), - flagSet.StringVarP(&options.StoreResponseDir,"store-response-dir", "srd", "output", "Custom directory to store HTTP responses"), + flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "output", "Custom directory to store HTTP responses"), flagSet.BoolVar(&options.JSONOutput, "json", false, "Output in JSONL(ines) format"), - flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr",false, "Include HTTP request/response in JSON output (-json only)"), + flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "Include HTTP request/response in JSON output (-json only)"), flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Include redirect HTTP Chain in JSON output (-json only)"), flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Include HTTP redirect chain in responses (-sr only)"), flagSet.BoolVar(&options.CSVOutput, "csv", false, "Output in CSV format"), - ) createGroup(flagSet, "configs", "Configurations", - flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "Max response size to save in bytes"), - flagSet.IntVarP(&options.MaxResponseBodySizeToRead,"response-size-to-read", "rstr", math.MaxInt32, "Max response size to read in bytes"), flagSet.Var(&options.Allow, "allow", "Allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "Denied list of IP/CIDR's to process (file or comma separated)"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), - flagSet.VarP(&options.CustomHeaders,"header", "H", "Custom Header to send with request"), - flagSet.StringVarP(&options.HTTPProxy,"proxy", "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080"), + flagSet.VarP(&options.CustomHeaders, "header", "H", "Custom Header to send with request"), + flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080"), flagSet.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization"), flagSet.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg"), - flagSet.BoolVarP(&options.NoColor,"no-color", "nc", false, "Disable color in output"), - flagSet.BoolVarP(&options.NoFallbackScheme,"no-fallback-scheme", "nfs", false, "Probe with input protocol scheme"), - flagSet.BoolVarP(&options.FollowRedirects,"follow-redirects", "fr", false, "Follow HTTP redirects"), - flagSet.BoolVarP(&options.FollowHostRedirects,"follow-host-redirects","fhr", false, "Follow redirects on the same host"), - flagSet.IntVarP(&options.MaxRedirects,"max-redirects","maxr", 10, "Max number of redirects to follow per host"), + flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "Follow HTTP redirects"), + flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "maxr", 10, "Max number of redirects to follow per host"), + flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "Follow redirects on the same host"), flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input"), flagSet.StringVar(&options.Methods, "x", "", "Request methods to use, use 'all' to probe all HTTP methods"), flagSet.StringVar(&options.RequestBody, "body", "", "Post body to include in HTTP request"), - flagSet.BoolVarP(&options.Stream, "stream","s", false, "Stream mode - start elaborating input targets without sorting"), - flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe","sd", false, "Disable dedupe input items (only used with stream mode)"), + flagSet.BoolVarP(&options.Stream, "stream", "s", false, "Stream mode - start elaborating input targets without sorting"), + flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "Disable dedupe input items (only used with stream mode)"), + flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "Probe all the ips associated with same host"), ) createGroup(flagSet, "debug", "Debug", flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"), - flagSet.BoolVar(&options.Verbose, "verbose", false, "Verbose mode"), + flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "Verbose mode"), flagSet.BoolVar(&options.Version, "version", false, "Display version"), + flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable color in output"), flagSet.BoolVar(&options.Debug, "debug", false, "Debug mode"), flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "Show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "Show all received responses"), @@ -303,10 +301,14 @@ func ParseOptions() *Options { ) createGroup(flagSet, "Optimizations", "Optimizations", + flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "Display both probbed protocol (HTTPS and HTTP)"), + flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "Probe with input protocol scheme"), + flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "Max error count per host before skipping remaining path/s"), + flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "Skip full port scans for CDNs (only checks for 80,443)"), flagSet.IntVar(&options.Retries, "retries", 0, "Number of retries"), flagSet.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds"), - flagSet.IntVarP(&options.HostMaxErrors,"max-host-error", "maxhr", 30, "Max error count per host before skipping remaining path/s"), - flagSet.BoolVarP(&options.ExcludeCDN,"exclude-cdn", "ec", false, "Skip full port scans for CDNs (only checks for 80,443)"), + flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "Max response size to save in bytes"), + flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "Max response size to read in bytes"), ) _ = flagSet.Parse() diff --git a/runner/runner.go b/runner/runner.go index f743838..1536102 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -3,6 +3,7 @@ package runner import ( "bufio" "bytes" + "context" "crypto/sha256" "encoding/csv" "encoding/hex" @@ -28,6 +29,7 @@ import ( "github.com/projectdiscovery/clistats" "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/goconfig" + "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" @@ -223,6 +225,7 @@ func New(options *Options) (*Runner, error) { scanopts.ExcludeCDN = options.ExcludeCDN scanopts.HostMaxErrors = options.HostMaxErrors + scanopts.ProbeAllIPS = options.ProbeAllIPS runner.scanopts = scanopts if options.ShowStatistics { @@ -637,7 +640,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. protocols = []string{httpx.HTTPS, httpx.HTTP} } - for target := range targets(stringz.TrimProtocol(t, scanopts.NoFallback || scanopts.NoFallbackScheme)) { + for target := range r.targets(hp, stringz.TrimProtocol(t, scanopts.NoFallback || scanopts.NoFallbackScheme)) { // if no custom ports specified then test the default ones if len(customport.Ports) == 0 { for _, method := range scanopts.Methods { @@ -700,7 +703,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. } // returns all the targets within a cidr range or the single target -func targets(target string) chan string { +func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { results := make(chan string) go func() { defer close(results) @@ -721,6 +724,18 @@ func targets(target string) chan string { for _, ip := range cidrIps { results <- ip } + } else if r.options.ProbeAllIPS { + URL, err := urlutil.Parse(target) + if err != nil { + results <- target + } + ips, _, err := getDNSData(hp, URL.Host) + if err != nil || len(ips) == 0 { + results <- target + } + for _, ip := range ips { + results <- strings.Join([]string{ip, target}, ",") + } } else { results <- target } @@ -735,7 +750,14 @@ func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method, origInput st } retried := false retry: - var customHost string + var customHost, customIP string + if scanopts.ProbeAllIPS { + parts := strings.SplitN(domain, ",", 2) + if len(parts) == 2 { + customIP = parts[0] + domain = parts[1] + } + } if scanopts.VHostInput { parts := strings.Split(domain, ",") //nolint:gomnd // not a magic number @@ -781,7 +803,14 @@ retry: // in case of standard requests append the new path to the existing one URL.RequestURI += scanopts.RequestURI } - req, err := hp.NewRequest(method, URL.String()) + var req *retryablehttp.Request + if customIP != "" { + customHost = URL.Host + ctx := context.WithValue(context.Background(), "ip", customIP) //nolint + req, err = hp.NewRequestWithContext(ctx, method, URL.String()) + } else { + req, err = hp.NewRequest(method, URL.String()) + } if err != nil { return Result{URL: URL.String(), Input: origInput, err: err} } @@ -1048,20 +1077,16 @@ retry: } } ip := hp.Dialer.GetDialedIP(URL.Host) - if scanopts.OutputIP { + // hp.Dialer.GetDialedIP would return only the last dialed one + if customIP != "" { + ip = customIP + } + if scanopts.OutputIP || scanopts.ProbeAllIPS { builder.WriteString(fmt.Sprintf(" [%s]", ip)) } - var ( - ips []string - cnames []string - ) - dnsData, err := hp.Dialer.GetDNSData(URL.Host) - if dnsData != nil && err == nil { - ips = append(ips, dnsData.A...) - ips = append(ips, dnsData.AAAA...) - cnames = dnsData.CNAME - } else { + ips, cnames, err := getDNSData(hp, domain) + if err != nil { ips = append(ips, ip) } @@ -1379,3 +1404,15 @@ func (r *Runner) skipCDNPort(host string, port string) bool { return false } + +func getDNSData(hp *httpx.HTTPX, hostname string) (ips, cnames []string, err error) { + dnsData, err := hp.Dialer.GetDNSData(hostname) + if err != nil { + return nil, nil, err + } + ips = make([]string, 0, len(dnsData.A)+len(dnsData.AAAA)) + ips = append(ips, dnsData.A...) + ips = append(ips, dnsData.AAAA...) + cnames = dnsData.CNAME + return +} From 5d61e58ca0a43bc4a7c76c07b001ae0ca19bc91d Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 1 Dec 2021 12:44:00 +0530 Subject: [PATCH 064/355] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index ecff256..f432f57 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.4-dev + /_/ v1.1.4 ` // Version is the current version of httpx -const Version = `v1.1.4-dev` +const Version = `v1.1.4` // showBanner is used to show the banner to the user func showBanner() { From ff6ff5191e99d310dab61449ca5777b2703848cc Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 1 Dec 2021 12:54:12 +0530 Subject: [PATCH 065/355] readme update --- README.md | 63 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 7b98b05..668ae42 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,6 @@ PROBES: -cname Display Host cname -cdn Display if CDN in use -probe Display probe status - -nf, -no-fallback Display both protocol (HTTPS and HTTP) MATCHERS: -mc, -match-code string Match response with given status code (-mc 200,302) @@ -139,40 +138,42 @@ OUTPUT: -csv Output in CSV format CONFIGURATIONS: - -rsts, -response-size-to-save int Max response size to save in bytes (default 2147483647) - -rstr, -response-size-to-read int Max response size to read in bytes (default 2147483647) - -allow string[] Allowed list of IP/CIDR's to process (file or comma separated) - -deny string[] Denied list of IP/CIDR's to process (file or comma separated) - -random-agent Enable Random User-Agent to use (default true) - -H, -header string[] Custom Header to send with request - -http-proxy, -proxy string HTTP Proxy, eg http://127.0.0.1:8080 - -unsafe Send raw requests skipping golang normalization - -resume Resume scan using resume.cfg - -nc, -no-color Disable color in output - -nfs, -no-fallback-scheme Probe with input protocol scheme - -fr, -follow-redirects Follow HTTP redirects - -fhr, -follow-host-redirects Follow redirects on the same host - -maxr, -max-redirects int Max number of redirects to follow per host (default 10) - -vhost-input Get a list of vhosts as input - -x string Request methods to use, use 'all' to probe all HTTP methods - -body string Post body to include in HTTP request - -s, -stream Stream mode - start elaborating input targets without sorting - -sd, -skip-dedupe Disable dedupe input items (only used with stream mode) + -allow string[] Allowed list of IP/CIDR's to process (file or comma separated) + -deny string[] Denied list of IP/CIDR's to process (file or comma separated) + -random-agent Enable Random User-Agent to use (default true) + -H, -header string[] Custom Header to send with request + -http-proxy, -proxy string HTTP Proxy, eg http://127.0.0.1:8080 + -unsafe Send raw requests skipping golang normalization + -resume Resume scan using resume.cfg + -fr, -follow-redirects Follow HTTP redirects + -maxr, -max-redirects int Max number of redirects to follow per host (default 10) + -fhr, -follow-host-redirects Follow redirects on the same host + -vhost-input Get a list of vhosts as input + -x string Request methods to use, use 'all' to probe all HTTP methods + -body string Post body to include in HTTP request + -s, -stream Stream mode - start elaborating input targets without sorting + -sd, -skip-dedupe Disable dedupe input items (only used with stream mode) + -pa, -probe-all-ips Probe all the ips associated with same host DEBUG: - -silent Silent mode - -verbose Verbose mode - -version Display version - -debug Debug mode - -debug-req Show all sent requests - -debug-resp Show all received responses - -stats Display scan statistic + -silent Silent mode + -v, -verbose Verbose mode + -version Display version + -nc, -no-color Disable color in output + -debug Debug mode + -debug-req Show all sent requests + -debug-resp Show all received responses + -stats Display scan statistic OPTIMIZATIONS: - -retries int Number of retries - -timeout int Timeout in seconds (default 5) - -maxhr, -max-host-error int Max error count per host before skipping remaining path/s (default 30) - -ec, -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) + -nf, -no-fallback Display both probbed protocol (HTTPS and HTTP) + -nfs, -no-fallback-scheme Probe with input protocol scheme + -maxhr, -max-host-error int Max error count per host before skipping remaining path/s (default 30) + -ec, -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) + -retries int Number of retries + -timeout int Timeout in seconds (default 5) + -rsts, -response-size-to-save int Max response size to save in bytes (default 2147483647) + -rstr, -response-size-to-read int Max response size to read in bytes (default 2147483647) ``` # Running httpX From 8e9e96c08503891302371a71ca09b0275f7607a4 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 3 Dec 2021 06:59:31 +0530 Subject: [PATCH 066/355] dev version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index f432f57..97ac18c 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.4 + /_/ v1.1.5-dev ` // Version is the current version of httpx -const Version = `v1.1.4` +const Version = `v1.1.5-dev` // showBanner is used to show the banner to the user func showBanner() { From a83eb839d08fe0bf9f083d3818bc1818c061dbff Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Mon, 6 Dec 2021 14:31:32 +0530 Subject: [PATCH 067/355] while probing remove wildcard from domain if detected and then process --- runner/runner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 1536102..130ac8b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -712,7 +712,8 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { // * // spaces if strings.ContainsAny(target, " *") { - return + // trim *. from the target to return the domain instead of wildard + target = strings.Trim(target, "*.") } // test if the target is a cidr From 6f67a1c301cbf93f0977f18df501db636e6d55d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Dec 2021 10:05:54 +0000 Subject: [PATCH 068/355] chore(deps): bump golang from 1.17.3-alpine to 1.17.4-alpine Bumps golang from 1.17.3-alpine to 1.17.4-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d16b9ec..69f14b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.3-alpine AS builder +FROM golang:1.17.4-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 0e94ba14bdb48543afc3031d969857a05ba949c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Dec 2021 10:07:41 +0000 Subject: [PATCH 069/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.18 to 0.0.19. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.18...v0.0.19) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 93d9a11..bc580f7 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.18 + github.com/projectdiscovery/wappalyzergo v0.0.19 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 36b9d10..36e1a20 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.18 h1:R2BcCFE777QEtON+mnKyVbD4qSIu35Ppe0UPR7kDw3s= -github.com/projectdiscovery/wappalyzergo v0.0.18/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.19 h1:7na+MInmuAAA3NxA6tbO6TIeBuOAbUbc1525pcaU6mE= +github.com/projectdiscovery/wappalyzergo v0.0.19/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 2a05756f8b558b90489c2720e6934179cbb6005d Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Mon, 6 Dec 2021 17:51:14 +0530 Subject: [PATCH 070/355] trim prefix . from target ##447 --- runner/runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 130ac8b..610124c 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -711,9 +711,9 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { // A valid target does not contain: // * // spaces - if strings.ContainsAny(target, " *") { - // trim *. from the target to return the domain instead of wildard - target = strings.Trim(target, "*.") + if strings.ContainsAny(target, "*") || strings.HasPrefix(target, ".") { + // trim * and/or . (prefix) from the target to return the domain instead of wildard + target = strings.TrimPrefix(strings.Trim(target, "*"), ".") } // test if the target is a cidr From b0b3ae00749e2c4c054548cdbb85f316ed3b21a5 Mon Sep 17 00:00:00 2001 From: Sajad Parra Date: Tue, 7 Dec 2021 15:53:23 +0530 Subject: [PATCH 071/355] add check to ignore duplicate target #447 --- runner/runner.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 610124c..e997aa6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -653,15 +653,24 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false for _, tt := range result.TLSData.DNSNames { + if !r.testAndSet(tt) { + continue + } r.process(tt, wg, hp, protocol, scanopts, output) } for _, tt := range result.TLSData.CommonName { + if !r.testAndSet(tt) { + continue + } r.process(tt, wg, hp, protocol, scanopts, output) } } if scanopts.CSPProbe && result.CSPData != nil { scanopts.CSPProbe = false for _, tt := range result.CSPData.Domains { + if !r.testAndSet(tt) { + continue + } r.process(tt, wg, hp, protocol, scanopts, output) } } @@ -686,9 +695,15 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false for _, tt := range result.TLSData.DNSNames { + if !r.testAndSet(tt) { + continue + } r.process(tt, wg, hp, protocol, scanopts, output) } for _, tt := range result.TLSData.CommonName { + if !r.testAndSet(tt) { + continue + } r.process(tt, wg, hp, protocol, scanopts, output) } } @@ -711,9 +726,12 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { // A valid target does not contain: // * // spaces - if strings.ContainsAny(target, "*") || strings.HasPrefix(target, ".") { + if strings.ContainsAny(target, "*") || strings.HasPrefix(target, ".") { // trim * and/or . (prefix) from the target to return the domain instead of wildard target = strings.TrimPrefix(strings.Trim(target, "*"), ".") + if !r.testAndSet(target) { + return + } } // test if the target is a cidr From cad469784d40816470c38a7ae76c909bf41fdd18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 10:06:08 +0000 Subject: [PATCH 072/355] chore(deps): bump golang from 1.17.4-alpine to 1.17.5-alpine Bumps golang from 1.17.4-alpine to 1.17.5-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 69f14b8..2c91023 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.4-alpine AS builder +FROM golang:1.17.5-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 9b1b349c80610f0d0c78104bdfe48b00fe707a2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 10:07:58 +0000 Subject: [PATCH 073/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.19 to 0.0.20. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.19...v0.0.20) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bc580f7..3243979 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.19 + github.com/projectdiscovery/wappalyzergo v0.0.20 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 36e1a20..c4bfe2e 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.19 h1:7na+MInmuAAA3NxA6tbO6TIeBuOAbUbc1525pcaU6mE= -github.com/projectdiscovery/wappalyzergo v0.0.19/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.20 h1:6gP5jzlWA6lbxiDM7P7k+ECOdZrYBvyXV6Y8i4KkewA= +github.com/projectdiscovery/wappalyzergo v0.0.20/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 0abc50da7679c70c84fa8080613ff7abdfd12f5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Dec 2021 10:06:37 +0000 Subject: [PATCH 074/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.20 to 0.0.21. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.20...v0.0.21) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3243979..a53a481 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.20 + github.com/projectdiscovery/wappalyzergo v0.0.21 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index c4bfe2e..eed5fff 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.20 h1:6gP5jzlWA6lbxiDM7P7k+ECOdZrYBvyXV6Y8i4KkewA= -github.com/projectdiscovery/wappalyzergo v0.0.20/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.21 h1:M+XKwGc5Vh/kXYsjwd8Z/IVE85D7XL2BRScU7jmKDqM= +github.com/projectdiscovery/wappalyzergo v0.0.21/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 9fb26a9dbf92372bb0a266d3fab07c727f733f0d Mon Sep 17 00:00:00 2001 From: ola456 Date: Wed, 22 Dec 2021 12:48:44 +0100 Subject: [PATCH 075/355] fix truncation of long response file names --- runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index e997aa6..6ee93f6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1173,9 +1173,9 @@ retry: // On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux // Truncating length at 255 - if len(domainFile) >= maxFileNameLength { + if len(domainFile) >= maxFileNameLength-4 { // leaving last 4 bytes free to append ".txt" - domainFile = domainFile[:maxFileNameLength-1] + domainFile = domainFile[:maxFileNameLength-4] } domainFile = strings.ReplaceAll(domainFile, "/", "_") + ".txt" From 0277ca1c847b982ff2da2e63018e82f08cb1b6f2 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 23 Dec 2021 19:48:53 +0100 Subject: [PATCH 076/355] hardcoding offset into constant for better performances --- runner/options.go | 3 ++- runner/runner.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/runner/options.go b/runner/options.go index 8bb3347..210f97b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -19,7 +19,8 @@ import ( ) const ( - maxFileNameLength = 255 + // The maximum file length is 251 (255 - 4 bytes for ".ext" suffix) + maxFileNameLength = 251 two = 2 DefaultResumeFile = "resume.cfg" ) diff --git a/runner/runner.go b/runner/runner.go index 6ee93f6..9dc4131 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1173,9 +1173,9 @@ retry: // On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux // Truncating length at 255 - if len(domainFile) >= maxFileNameLength-4 { + if len(domainFile) >= maxFileNameLength { // leaving last 4 bytes free to append ".txt" - domainFile = domainFile[:maxFileNameLength-4] + domainFile = domainFile[:maxFileNameLength] } domainFile = strings.ReplaceAll(domainFile, "/", "_") + ".txt" From f15476ef259db4714a8692b1fbbf1ecf5548c7da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Dec 2021 10:05:28 +0000 Subject: [PATCH 077/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.21 to 0.0.22. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.21...v0.0.22) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a53a481..bd8a18d 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.21 + github.com/projectdiscovery/wappalyzergo v0.0.22 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index eed5fff..f5152c8 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.21 h1:M+XKwGc5Vh/kXYsjwd8Z/IVE85D7XL2BRScU7jmKDqM= -github.com/projectdiscovery/wappalyzergo v0.0.21/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.22 h1:nPEl0eUfYvAUH9xEZPF2kuboh92DZJQz1G6pbLS7XdU= +github.com/projectdiscovery/wappalyzergo v0.0.22/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 96a511ad4806d545d44f1fd5eb7d5919dd071f70 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 28 Dec 2021 07:46:42 +0100 Subject: [PATCH 078/355] Adding support for custom resolvers --- common/httpx/httpx.go | 3 +++ common/httpx/option.go | 1 + runner/options.go | 23 +++++++++++++++++++++++ runner/runner.go | 1 + 4 files changed, 28 insertions(+) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index b7c84d2..a0cc15d 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -44,6 +44,9 @@ func New(options *Options) (*HTTPX, error) { fastdialerOpts.Deny = options.Deny fastdialerOpts.Allow = options.Allow fastdialerOpts.WithDialerHistory = true + if len(options.Resolvers) > 0 { + fastdialerOpts.BaseResolvers = options.Resolvers + } dialer, err := fastdialer.NewDialer(fastdialerOpts) if err != nil { return nil, fmt.Errorf("could not create resolver cache: %s", err) diff --git a/common/httpx/option.go b/common/httpx/option.go index 46d6070..36d834a 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -36,6 +36,7 @@ type Options struct { MaxResponseBodySizeToSave int64 MaxResponseBodySizeToRead int64 UnsafeURI string + Resolvers []string } // DefaultOptions contains the default options diff --git a/runner/options.go b/runner/options.go index 210f97b..7631aa5 100644 --- a/runner/options.go +++ b/runner/options.go @@ -4,6 +4,7 @@ import ( "math" "os" "regexp" + "strings" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" @@ -197,6 +198,7 @@ type Options struct { Stream bool SkipDedupe bool ProbeAllIPS bool + Resolvers goflags.NormalizedStringSlice } // ParseOptions parses the command line options for application @@ -288,6 +290,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.Stream, "stream", "s", false, "Stream mode - start elaborating input targets without sorting"), flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "Disable dedupe input items (only used with stream mode)"), flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "Probe all the ips associated with same host"), + flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "List of resolvers (file or comma separated) - each resolver can be in the form protocol:ip|hostname:port - invalid files/resolvers ignored"), ) createGroup(flagSet, "debug", "Debug", @@ -370,6 +373,26 @@ func (options *Options) validateOptions() { gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err) } } + + var resolvers []string + for _, resolver := range options.Resolvers { + if fileutil.FileExists(resolver) { + chFile, err := fileutil.ReadFile(resolver) + if err != nil { + gologger.Fatal().Msgf("Couldn't process resolver file \"%s\": %s\n", resolver, err) + } + for line := range chFile { + resolvers = append(resolvers, line) + } + } else { + resolvers = append(resolvers, resolver) + } + } + options.Resolvers = resolvers + if len(options.Resolvers) > 0 { + gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ",")) + + } } // configureOutput configures the output on the screen diff --git a/runner/runner.go b/runner/runner.go index 9dc4131..98112e3 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -108,6 +108,7 @@ func New(options *Options) (*Runner, error) { if httpxOptions.MaxResponseBodySizeToSave > httpxOptions.MaxResponseBodySizeToRead { httpxOptions.MaxResponseBodySizeToSave = httpxOptions.MaxResponseBodySizeToRead } + httpxOptions.Resolvers = options.Resolvers var key, value string httpxOptions.CustomHeaders = make(map[string]string) From 481716ea007302a86384184effdb50322fee4547 Mon Sep 17 00:00:00 2001 From: sandeep Date: Wed, 29 Dec 2021 15:31:47 +0530 Subject: [PATCH 079/355] readme update --- README.md | 4 +++- runner/options.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 668ae42..53f503d 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ OUTPUT: -csv Output in CSV format CONFIGURATIONS: + -r, -resolvers string[] List of custom resolvers (file or comma separated) -allow string[] Allowed list of IP/CIDR's to process (file or comma separated) -deny string[] Denied list of IP/CIDR's to process (file or comma separated) -random-agent Enable Random User-Agent to use (default true) @@ -323,7 +324,8 @@ https://support.hackerone.com - `vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes. - Unique flags should be used for specific use cases instead of running them as default with other flags. - When using `json` flag, all the information (default probes) included in the JSON output. - +- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (eg **udp:127.0.0.1:53**) +- Invalid custom resolvers/files are ignored. # Acknowledgement diff --git a/runner/options.go b/runner/options.go index 7631aa5..550f240 100644 --- a/runner/options.go +++ b/runner/options.go @@ -274,6 +274,7 @@ func ParseOptions() *Options { ) createGroup(flagSet, "configs", "Configurations", + flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "List of custom resolvers (file or comma separated)"), flagSet.Var(&options.Allow, "allow", "Allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "Denied list of IP/CIDR's to process (file or comma separated)"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), @@ -290,7 +291,6 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.Stream, "stream", "s", false, "Stream mode - start elaborating input targets without sorting"), flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "Disable dedupe input items (only used with stream mode)"), flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "Probe all the ips associated with same host"), - flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "List of resolvers (file or comma separated) - each resolver can be in the form protocol:ip|hostname:port - invalid files/resolvers ignored"), ) createGroup(flagSet, "debug", "Debug", From 47bbacf8779b7e70264fe20b8150f94e49eccda2 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 31 Dec 2021 09:26:39 +0100 Subject: [PATCH 080/355] Adding support for lines/words matchers/filters --- runner/options.go | 34 +++++++++++++++++++++++++++++++++- runner/runner.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 550f240..118239b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -66,6 +66,8 @@ type scanOptions struct { ExcludeCDN bool HostMaxErrors int ProbeAllIPS bool + OutputLinesCount bool + OutputWordsCount bool } func (s *scanOptions) Clone() *scanOptions { @@ -105,6 +107,8 @@ func (s *scanOptions) Clone() *scanOptions { MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, HostMaxErrors: s.HostMaxErrors, + OutputLinesCount: s.OutputLinesCount, + OutputWordsCount: s.OutputWordsCount, } } @@ -199,6 +203,16 @@ type Options struct { SkipDedupe bool ProbeAllIPS bool Resolvers goflags.NormalizedStringSlice + OutputLinesCount bool + OutputMatchLinesCount string + matchLinesCount []int + OutputFilterLinesCount string + filterLinesCount []int + OutputWordsCount bool + OutputMatchWordsCount string + matchWordsCount []int + OutputFilterWordsCount string + filterWordsCount []int } // ParseOptions parses the command line options for application @@ -228,6 +242,9 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"), flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), + flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), + flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, " Display Response body line count"), + flagSet.BoolVarP(&options.OutputWordsCount, "words-count", "wc", false, " Display Response body words count"), ) createGroup(flagSet, "matchers", "Matchers", @@ -235,7 +252,8 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"), flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"), - flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), + flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"), + flagSet.StringVarP(&options.OutputMatchWordsCount, "match-words-count", "mwc", "", "Match Response body words count"), ) createGroup(flagSet, "filters", "Filters", @@ -243,6 +261,8 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"), flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"), + flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"), + flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-words-count", "fwc", "", "Filter Response body words count"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", @@ -373,6 +393,18 @@ func (options *Options) validateOptions() { gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err) } } + if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil { + gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err) + } + if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil { + gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err) + } + if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil { + gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err) + } + if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil { + gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err) + } var resolvers []string for _, resolver := range options.Resolvers { diff --git a/runner/runner.go b/runner/runner.go index 98112e3..c717436 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -227,6 +227,8 @@ func New(options *Options) (*Runner, error) { scanopts.ExcludeCDN = options.ExcludeCDN scanopts.HostMaxErrors = options.HostMaxErrors scanopts.ProbeAllIPS = options.ProbeAllIPS + scanopts.OutputLinesCount = options.OutputLinesCount + scanopts.OutputWordsCount = options.OutputWordsCount runner.scanopts = scanopts if options.ShowStatistics { @@ -552,6 +554,12 @@ func (r *Runner) RunEnumeration() { if len(r.options.filterContentLength) > 0 && slice.IntSliceContains(r.options.filterContentLength, resp.ContentLength) { continue } + if len(r.options.filterLinesCount) > 0 && slice.IntSliceContains(r.options.filterLinesCount, resp.Lines) { + continue + } + if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) { + continue + } if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) { continue } @@ -570,6 +578,12 @@ func (r *Runner) RunEnumeration() { if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) { continue } + if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) { + continue + } + if len(r.options.matchWordsCount) > 0 && !slice.IntSliceContains(r.options.matchWordsCount, resp.Words) { + continue + } row := resp.str if r.options.JSONOutput { @@ -1168,6 +1182,26 @@ retry: builder.WriteRune(']') } + if scanopts.OutputLinesCount { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(resp.Lines).String()) + } else { + builder.WriteString(fmt.Sprint(resp.Lines)) + } + builder.WriteRune(']') + } + + if scanopts.OutputWordsCount { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(resp.Words).String()) + } else { + builder.WriteString(fmt.Sprint(resp.Words)) + } + builder.WriteRune(']') + } + // store responses or chain in directory if scanopts.StoreResponse || scanopts.StoreChain { domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".") @@ -1271,6 +1305,8 @@ retry: ResponseTime: resp.Duration.String(), Technologies: technologies, FinalURL: finalURL, + Lines: resp.Lines, + Words: resp.Words, } } @@ -1322,6 +1358,8 @@ type Result struct { Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` FinalURL string `json:"final-url,omitempty" csv:"final-url"` Failed bool `json:"failed" csv:"failed"` + Lines int `json:"lines" csv:"lines"` + Words int `json:"words" csv:"words"` } // JSON the result From 20254fd8425663caa8a9df3d34047d0aba2f3274 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 31 Dec 2021 16:45:40 +0530 Subject: [PATCH 081/355] readme updates --- README.md | 31 +++++++++++++++++++------------ runner/options.go | 8 ++++---- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 53f503d..8eb0399 100644 --- a/README.md +++ b/README.md @@ -43,18 +43,19 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi ### Supported probes:- -| Probes | Default check | Probes | Default check | -|--------------------|-----------------|--------------------|-----------------| -| URL | true | IP | true | -| Title | true | CNAME | true | -| Status Code | true | Raw HTTP | false | -| Content Length | true | HTTP2 | false | -| TLS Certificate | true | HTTP 1.1 Pipeline | false | -| CSP Header | true | Virtual host | false | -| Location Header | true | CDN | false | -| Web Server | true | Path | false | -| Web Socket | true | Ports | false | -| Response Time | true | Request method | false | +| Probes | Default check | Probes | Default check | +| --------------- | ------------- | ----------------- | ------------- | +| URL | true | IP | true | +| Title | true | CNAME | true | +| Status Code | true | Raw HTTP | false | +| Content Length | true | HTTP2 | false | +| Line Count | true | Word Count | true | +| TLS Certificate | true | HTTP 1.1 Pipeline | false | +| CSP Header | true | Virtual host | false | +| Location Header | true | CDN | false | +| Web Server | true | Paths | false | +| Web Socket | true | Ports | false | +| Response Time | true | Request Method | false | # Installation Instructions @@ -87,6 +88,8 @@ PROBES: -sc, -status-code Display Status Code -td, -tech-detect Display wappalyzer based technology detection -cl, -content-length Display Content-Length + -lc, -line-count Display Response body line count + -wc, -word-count Display Response body word count -server, -web-server Display Server header -ct, -content-type Display Content-Type header -rt, -response-time Display the response time @@ -105,12 +108,16 @@ MATCHERS: -ms, -match-string string Match response with given string -mr, -match-regex string Match response with specific regex -er, -extract-regex string Display response content with matched regex + -mlc, -match-line-count string Match Response body line count + -mwc, -match-word-count string Match Response body word count FILTERS: -fc, -filter-code string Filter response with given status code (-fc 403,401) -fl, -filter-length string Filter response with given content length (-fl 23,33) -fs, -filter-string string Filter response with specific string -fe, -filter-regex string Filter response with specific regex + -flc, -filter-line-count string Filter Response body line count + -fwc, -filter-word-count string Filter Response body word count RATE-LIMIT: -t, -threads int Number of threads (default 50) diff --git a/runner/options.go b/runner/options.go index 118239b..8af0455 100644 --- a/runner/options.go +++ b/runner/options.go @@ -233,6 +233,8 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "Display Content-Length"), flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "Display Server header"), flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "Display Content-Type header"), + flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "Display Response body line count"), + flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "Display Response body word count"), flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "Display the response time"), flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"), flagSet.BoolVar(&options.Location, "location", false, "Display Location header"), @@ -243,8 +245,6 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"), flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), - flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, " Display Response body line count"), - flagSet.BoolVarP(&options.OutputWordsCount, "words-count", "wc", false, " Display Response body words count"), ) createGroup(flagSet, "matchers", "Matchers", @@ -253,7 +253,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"), flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"), - flagSet.StringVarP(&options.OutputMatchWordsCount, "match-words-count", "mwc", "", "Match Response body words count"), + flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"), ) createGroup(flagSet, "filters", "Filters", @@ -262,7 +262,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"), flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"), - flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-words-count", "fwc", "", "Filter Response body words count"), + flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", From b4f2ac9b7f1ddc7cd3913d21c1e4c200964a1410 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 3 Jan 2022 08:53:15 +0100 Subject: [PATCH 082/355] Enabling store response automatically if output directory is specified --- runner/options.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runner/options.go b/runner/options.go index 550f240..441c023 100644 --- a/runner/options.go +++ b/runner/options.go @@ -391,7 +391,11 @@ func (options *Options) validateOptions() { options.Resolvers = resolvers if len(options.Resolvers) > 0 { gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ",")) + } + if options.StoreResponseDir != "" && !options.StoreResponse { + gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n") + options.StoreResponse = true } } From 27aef8ca00e36102cffb278463041dd9ac0289b4 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 3 Jan 2022 13:52:45 +0530 Subject: [PATCH 083/355] misc updates --- README.md | 16 ++++++++-------- runner/options.go | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 53f503d..c9531e0 100644 --- a/README.md +++ b/README.md @@ -128,14 +128,14 @@ MISCELLANEOUS: -paths string File or comma separated paths to request (deprecated) OUTPUT: - -o, -output string File to write output - -sr, -store-response Store HTTP responses - -srd, -store-response-dir string Custom directory to store HTTP responses (default "output") - -json Output in JSONL(ines) format - -irr, -include-response Include HTTP request/response in JSON output (-json only) - -include-chain Include redirect HTTP Chain in JSON output (-json only) - -store-chain Include HTTP redirect chain in responses (-sr only) - -csv Output in CSV format + -o, -output string file to write output + -sr, -store-response store http response to output directory + -srd, -store-response-dir string store http response to custom directory (default "output") + -csv store output in CSV format + -json store output in JSONL(ines) format + -irr, -include-response include http request/response in JSON output (-json only) + -include-chain include redirect http chain in JSON output (-json only) + -store-chain include http redirect chain in responses (-sr only) CONFIGURATIONS: -r, -resolvers string[] List of custom resolvers (file or comma separated) diff --git a/runner/options.go b/runner/options.go index 441c023..68c1068 100644 --- a/runner/options.go +++ b/runner/options.go @@ -263,14 +263,14 @@ func ParseOptions() *Options { ) createGroup(flagSet, "output", "Output", - flagSet.StringVarP(&options.Output, "output", "o", "", "File to write output"), - flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "Store HTTP responses"), - flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "output", "Custom directory to store HTTP responses"), - flagSet.BoolVar(&options.JSONOutput, "json", false, "Output in JSONL(ines) format"), - flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "Include HTTP request/response in JSON output (-json only)"), - flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "Include redirect HTTP Chain in JSON output (-json only)"), - flagSet.BoolVar(&options.StoreChain, "store-chain", false, "Include HTTP redirect chain in responses (-sr only)"), - flagSet.BoolVar(&options.CSVOutput, "csv", false, "Output in CSV format"), + flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"), + flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"), + flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "output", "store http response to custom directory"), + flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in CSV format"), + flagSet.BoolVar(&options.JSONOutput, "json", false, "store output in JSONL(ines) format"), + flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"), + flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"), + flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"), ) createGroup(flagSet, "configs", "Configurations", From eaccae320ae80d3dc64e0a4ce87edf5ee795fc67 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 3 Jan 2022 09:27:04 +0100 Subject: [PATCH 084/355] Adding CLI option to trim default HTTP/HTTPS ports from Host header --- runner/options.go | 4 ++++ runner/runner.go | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/runner/options.go b/runner/options.go index 550f240..1aaf5d3 100644 --- a/runner/options.go +++ b/runner/options.go @@ -66,6 +66,7 @@ type scanOptions struct { ExcludeCDN bool HostMaxErrors int ProbeAllIPS bool + LeaveDefaultPorts bool } func (s *scanOptions) Clone() *scanOptions { @@ -105,6 +106,7 @@ func (s *scanOptions) Clone() *scanOptions { MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, HostMaxErrors: s.HostMaxErrors, + LeaveDefaultPorts: s.LeaveDefaultPorts, } } @@ -199,6 +201,7 @@ type Options struct { SkipDedupe bool ProbeAllIPS bool Resolvers goflags.NormalizedStringSlice + LeaveDefaultPorts bool } // ParseOptions parses the command line options for application @@ -291,6 +294,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.Stream, "stream", "s", false, "Stream mode - start elaborating input targets without sorting"), flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "Disable dedupe input items (only used with stream mode)"), flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "Probe all the ips associated with same host"), + flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "Leave default HTTP/HTTPS ports (eg. http://host:80 - https//host:443"), ) createGroup(flagSet, "debug", "Debug", diff --git a/runner/runner.go b/runner/runner.go index 98112e3..d59d37d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -227,6 +227,7 @@ func New(options *Options) (*Runner, error) { scanopts.ExcludeCDN = options.ExcludeCDN scanopts.HostMaxErrors = options.HostMaxErrors scanopts.ProbeAllIPS = options.ProbeAllIPS + scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts runner.scanopts = scanopts if options.ShowStatistics { @@ -839,6 +840,15 @@ retry: req.Host = customHost } + if !scanopts.LeaveDefaultPorts { + switch { + case protocol == httpx.HTTP && strings.HasSuffix(req.Host, ":80"): + req.Host = strings.TrimSuffix(req.Host, ":80") + case protocol == httpx.HTTPS && strings.HasSuffix(req.Host, ":443"): + req.Host = strings.TrimSuffix(req.Host, ":443") + } + } + hp.SetCustomHeaders(req, hp.CustomHeaders) // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { From 6c65703edfad16b820a24b5e396d331368ffffc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:07:50 +0000 Subject: [PATCH 085/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.22 to 0.0.23. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.22...v0.0.23) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bd8a18d..4733306 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.22 + github.com/projectdiscovery/wappalyzergo v0.0.23 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index f5152c8..538d57e 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.22 h1:nPEl0eUfYvAUH9xEZPF2kuboh92DZJQz1G6pbLS7XdU= -github.com/projectdiscovery/wappalyzergo v0.0.22/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.23 h1:ciuuPOPvkDjRXIzxTXtIp/nOhpE/l2tne33FpXHaMJA= +github.com/projectdiscovery/wappalyzergo v0.0.23/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 761c7ac9b88009aca0ae53ec8b10d6cb98525ffa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jan 2022 10:08:03 +0000 Subject: [PATCH 086/355] chore(deps): bump github.com/microcosm-cc/bluemonday Bumps [github.com/microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday) from 1.0.16 to 1.0.17. - [Release notes](https://github.com/microcosm-cc/bluemonday/releases) - [Commits](https://github.com/microcosm-cc/bluemonday/compare/v1.0.16...v1.0.17) --- updated-dependencies: - dependency-name: github.com/microcosm-cc/bluemonday dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bd8a18d..4ad364b 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/microcosm-cc/bluemonday v1.0.16 + github.com/microcosm-cc/bluemonday v1.0.17 github.com/miekg/dns v1.1.43 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.2 diff --git a/go.sum b/go.sum index f5152c8..a069166 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc= -github.com/microcosm-cc/bluemonday v1.0.16/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/microcosm-cc/bluemonday v1.0.17 h1:Z1a//hgsQ4yjC+8zEkV8IWySkXnsxmdSY642CTFQb5Y= +github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= From 9b7189326afa1fcd915564d92c591186da759240 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 4 Jan 2022 09:47:06 +0100 Subject: [PATCH 087/355] improving output directory flag handling --- runner/options.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/runner/options.go b/runner/options.go index 8a32a95..39c44bc 100644 --- a/runner/options.go +++ b/runner/options.go @@ -21,9 +21,10 @@ import ( const ( // The maximum file length is 251 (255 - 4 bytes for ".ext" suffix) - maxFileNameLength = 251 - two = 2 - DefaultResumeFile = "resume.cfg" + maxFileNameLength = 251 + two = 2 + DefaultResumeFile = "resume.cfg" + DefaultOutputDirectory = "output" ) type scanOptions struct { @@ -288,7 +289,7 @@ func ParseOptions() *Options { createGroup(flagSet, "output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"), - flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "output", "store http response to custom directory"), + flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"), flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in CSV format"), flagSet.BoolVar(&options.JSONOutput, "json", false, "store output in JSONL(ines) format"), flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"), @@ -429,6 +430,10 @@ func (options *Options) validateOptions() { gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ",")) } + if options.StoreResponse && options.StoreResponseDir == "" { + gologger.Debug().Msgf("Store response directory not specified, using \"%s\"\n", DefaultOutputDirectory) + options.StoreResponseDir = DefaultOutputDirectory + } if options.StoreResponseDir != "" && !options.StoreResponse { gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n") options.StoreResponse = true From bee964315c0ddb48774282f83cf9d6b48db82deb Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 7 Jan 2022 12:58:07 +0100 Subject: [PATCH 088/355] Add support for favicon hash (#476) * Add support for favicon hash Co-authored-by: sandeep --- README.md | 7 +++++-- common/slice/slice.go | 10 +++++++++ common/stringz/stringz.go | 43 +++++++++++++++++++++++++++++++++++++++ go.mod | 2 ++ go.sum | 2 ++ runner/options.go | 15 +++++++++++++- runner/runner.go | 21 +++++++++++++++++++ 7 files changed, 97 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 988af0b..d8d103b 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ MATCHERS: -er, -extract-regex string Display response content with matched regex -mlc, -match-line-count string Match Response body line count -mwc, -match-word-count string Match Response body word count + -mfc, -match-favicon string[] Match response with specific favicon FILTERS: -fc, -filter-code string Filter response with given status code (-fc 403,401) @@ -118,12 +119,14 @@ FILTERS: -fe, -filter-regex string Filter response with specific regex -flc, -filter-line-count string Filter Response body line count -fwc, -filter-word-count string Filter Response body word count + -ffc, -filter-favicon string[] Filter response with specific favicon RATE-LIMIT: -t, -threads int Number of threads (default 50) -rl, -rate-limit int Maximum requests to send per second (default 150) MISCELLANEOUS: + -favicon Probes for favicon ("favicon.ico" as path) and display phythonic hash -tls-grab Perform TLS(SSL) data grabbing -tls-probe Send HTTP probes on the extracted TLS domains -csp-probe Send HTTP probes on the extracted CSP domains @@ -328,8 +331,8 @@ https://support.hackerone.com - As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable. - For printing both HTTP/HTTPS results, `no-fallback` flag can be used. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` -- `vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes. -- Unique flags should be used for specific use cases instead of running them as default with other flags. +- `favicon`,`vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes. +- Unique flags should be used for specific use cases instead of running them as default with other probes. - When using `json` flag, all the information (default probes) included in the JSON output. - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (eg **udp:127.0.0.1:53**) - Invalid custom resolvers/files are ignored. diff --git a/common/slice/slice.go b/common/slice/slice.go index 0102490..2426d84 100644 --- a/common/slice/slice.go +++ b/common/slice/slice.go @@ -10,6 +10,16 @@ func IntSliceContains(sl []int, v int) bool { return false } +// UIntSliceContains check if a slice contains the specified uint value +func UInt32SliceContains(sl []uint32, v uint32) bool { + for _, vv := range sl { + if vv == v { + return true + } + } + return false +} + // ToSlice creates a slice with all string keys from a map func ToSlice(m map[string]struct{}) (s []string) { for k := range m { diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index 34eb790..6c73201 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -1,11 +1,14 @@ package stringz import ( + "bytes" + "encoding/base64" "net/url" "strconv" "strings" "github.com/projectdiscovery/urlutil" + "github.com/spaolacci/murmur3" ) // TrimProtocol removes the HTTP scheme from an URI @@ -39,6 +42,24 @@ func StringToSliceInt(s string) ([]int, error) { return r, nil } +// StringToSliceUInt converts string to slice of ints +func StringToSliceUInt32(s string) ([]uint32, error) { + var r []uint32 + if s == "" { + return r, nil + } + for _, v := range strings.Split(s, ",") { + vTrim := strings.TrimSpace(v) + if i, err := strconv.ParseUint(vTrim, 10, 64); err == nil { + r = append(r, uint32(i)) + } else { + return r, err + } + } + + return r, nil +} + // SplitByCharAndTrimSpace splits string by a character and remove spaces func SplitByCharAndTrimSpace(s, splitchar string) (result []string) { for _, token := range strings.Split(s, splitchar) { @@ -84,3 +105,25 @@ func GetInvalidURI(rawURL string) (bool, string) { return false, "" } + +func FaviconHash(data []byte) int32 { + stdBase64 := base64.StdEncoding.EncodeToString(data) + stdBase64 = InsertInto(stdBase64, 76, '\n') + hasher := murmur3.New32WithSeed(0) + hasher.Write([]byte(stdBase64)) + return int32(hasher.Sum32()) +} + +func InsertInto(s string, interval int, sep rune) string { + var buffer bytes.Buffer + before := interval - 1 + last := len(s) - 1 + for i, char := range s { + buffer.WriteRune(char) + if i%interval == before && i != last { + buffer.WriteRune(sep) + } + } + buffer.WriteRune(sep) + return buffer.String() +} diff --git a/go.mod b/go.mod index 71fb315..7672f9b 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,8 @@ require ( golang.org/x/text v0.3.7 ) +require github.com/spaolacci/murmur3 v1.1.0 + require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 6b311c7..1a83f40 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,8 @@ github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsR github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/runner/options.go b/runner/options.go index 39c44bc..ca2f90e 100644 --- a/runner/options.go +++ b/runner/options.go @@ -67,6 +67,7 @@ type scanOptions struct { ExcludeCDN bool HostMaxErrors int ProbeAllIPS bool + Favicon bool LeaveDefaultPorts bool OutputLinesCount bool OutputWordsCount bool @@ -109,6 +110,7 @@ func (s *scanOptions) Clone() *scanOptions { MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave, MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, HostMaxErrors: s.HostMaxErrors, + Favicon: s.Favicon, LeaveDefaultPorts: s.LeaveDefaultPorts, OutputLinesCount: s.OutputLinesCount, OutputWordsCount: s.OutputWordsCount, @@ -206,6 +208,9 @@ type Options struct { SkipDedupe bool ProbeAllIPS bool Resolvers goflags.NormalizedStringSlice + Favicon bool + OutputFilterFavicon goflags.NormalizedStringSlice + OutputMatchFavicon goflags.NormalizedStringSlice LeaveDefaultPorts bool OutputLinesCount bool OutputMatchLinesCount string @@ -248,7 +253,6 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"), flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), - flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), ) createGroup(flagSet, "matchers", "Matchers", @@ -256,8 +260,10 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"), flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"), + flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"), flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"), + flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "Match response with specific favicon"), ) createGroup(flagSet, "filters", "Filters", @@ -267,6 +273,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"), flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"), flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"), + flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "Filter response with specific favicon"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", @@ -275,6 +282,7 @@ func ParseOptions() *Options { ) createGroup(flagSet, "Misc", "Miscellaneous", + flagSet.BoolVar(&options.Favicon, "favicon", false, "Probes for favicon (\"favicon.ico\" as path) and display phythonic hash"), flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"), flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"), flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"), @@ -438,6 +446,11 @@ func (options *Options) validateOptions() { gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n") options.StoreResponse = true } + + if options.Favicon { + gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n") + options.RequestURIs = "/favicon.ico" + } } // configureOutput configures the output on the screen diff --git a/runner/runner.go b/runner/runner.go index c3aee24..c8d8ee1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -227,6 +227,7 @@ func New(options *Options) (*Runner, error) { scanopts.ExcludeCDN = options.ExcludeCDN scanopts.HostMaxErrors = options.HostMaxErrors scanopts.ProbeAllIPS = options.ProbeAllIPS + scanopts.Favicon = options.Favicon scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts scanopts.OutputLinesCount = options.OutputLinesCount scanopts.OutputWordsCount = options.OutputWordsCount @@ -567,6 +568,9 @@ func (r *Runner) RunEnumeration() { if r.options.OutputFilterString != "" && strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputFilterString)) { continue } + if len(r.options.OutputFilterFavicon) > 0 && stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputFilterFavicon...) { + continue + } if len(r.options.matchStatusCode) > 0 && !slice.IntSliceContains(r.options.matchStatusCode, resp.StatusCode) { continue } @@ -579,6 +583,9 @@ func (r *Runner) RunEnumeration() { if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) { continue } + if len(r.options.OutputMatchFavicon) > 0 && !stringsutil.EqualFoldAny(resp.FavIconMMH3, r.options.OutputMatchFavicon...) { + continue + } if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) { continue } @@ -1192,6 +1199,18 @@ retry: builder.WriteRune(']') } + var faviconMMH3 string + if scanopts.Favicon { + faviconMMH3 = fmt.Sprintf("%d", stringz.FaviconHash(resp.Data)) + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(faviconMMH3).String()) + } else { + builder.WriteString(faviconMMH3) + } + builder.WriteRune(']') + } + if scanopts.OutputLinesCount { builder.WriteString(" [") if !scanopts.OutputWithNoColor { @@ -1315,6 +1334,7 @@ retry: ResponseTime: resp.Duration.String(), Technologies: technologies, FinalURL: finalURL, + FavIconMMH3: faviconMMH3, Lines: resp.Lines, Words: resp.Words, } @@ -1368,6 +1388,7 @@ type Result struct { Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` FinalURL string `json:"final-url,omitempty" csv:"final-url"` Failed bool `json:"failed" csv:"failed"` + FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` Lines int `json:"lines" csv:"lines"` Words int `json:"words" csv:"words"` } From 2a9da61ca6738214bebcdbd9d84959628568489f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 16:27:40 +0530 Subject: [PATCH 089/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#485) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.23 to 0.0.24. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.23...v0.0.24) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7672f9b..7af0f1d 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.23 + github.com/projectdiscovery/wappalyzergo v0.0.24 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1a83f40..879bcdb 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.23 h1:ciuuPOPvkDjRXIzxTXtIp/nOhpE/l2tne33FpXHaMJA= -github.com/projectdiscovery/wappalyzergo v0.0.23/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.24 h1:TQkJKawP+2/30uIG/n04imWtQ5SNt3qLvY7fOf1g0v4= +github.com/projectdiscovery/wappalyzergo v0.0.24/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 21a9bf7490e5ce44a72a15af8e1de7687b179f37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 16:27:52 +0530 Subject: [PATCH 090/355] chore(deps): bump golang from 1.17.5-alpine to 1.17.6-alpine (#484) Bumps golang from 1.17.5-alpine to 1.17.6-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2c91023..e18ce75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.5-alpine AS builder +FROM golang:1.17.6-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From f5935ebe612ca1c988403b215a2481d45e7b1a32 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 10 Jan 2022 12:11:47 +0100 Subject: [PATCH 091/355] Improving paths loading from file (#483) * Improving paths loading from file * improving invalid uri handling --- cmd/integration-test/http.go | 47 +++++++++++++++++++++++++++--------- runner/runner.go | 24 +++++++++++------- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 95a458d..e7b930e 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -12,18 +12,19 @@ import ( ) var httpTestcases = map[string]testutils.TestCase{ - "Standard HTTP GET Request": &standardHttpGet{}, - "Standard HTTPS GET Request": &standardHttpGet{tls: true}, - "Raw HTTP GET Request": &standardHttpGet{unsafe: true}, - "Raw request with non standard rfc path via stdin": &standardHttpGet{unsafe: true, stdinPath: "/%invalid"}, - "Raw request with non standard rfc path via cli flag": &standardHttpGet{unsafe: true, path: "/%invalid"}, - "Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect - "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output - "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin - "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body - "Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body - "Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path - "Regression test for: https://github.com/projectdiscovery/httpx/issues/433": &issue433{}, // new line scanning with title flag + "Standard HTTP GET Request": &standardHttpGet{}, + "Standard HTTPS GET Request": &standardHttpGet{tls: true}, + "Raw HTTP GET Request": &standardHttpGet{unsafe: true}, + "Raw request with non standard rfc path via stdin": &standardHttpGet{unsafe: true, stdinPath: "/%invalid"}, + "Raw request with non standard rfc path via cli flag": &standardHttpGet{unsafe: true, path: "/%invalid"}, + "Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect + "Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output + "Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin + "Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body + "Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body + "Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path + "Regression test for: https://github.com/projectdiscovery/httpx/issues/433": &issue433{}, // new line scanning with title flag + "Request URI to existing file - https://github.com/projectdiscovery/httpx/issues/480": &issue480{}, // request uri pointing to existing file } type standardHttpGet struct { @@ -264,3 +265,25 @@ func (h *issue433) Execute() error { } return nil } + +type issue480 struct{} + +func (h *issue480) Execute() error { + var ts *httptest.Server + router := httprouter.New() + uriPath := "////////////////../../../../../../../../etc/passwd" + router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + htmlResponse := "ok from uri" + fmt.Fprint(w, htmlResponse) + })) + ts = httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunHttpxAndGetResults(ts.URL, debug, "-path", "////////////////../../../../../../../../etc/passwd") + if err != nil { + return err + } + if !strings.Contains(results[0], uriPath) { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/runner/runner.go b/runner/runner.go index c8d8ee1..02814ad 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -16,6 +16,7 @@ import ( "net/url" "os" "path" + "path/filepath" "reflect" "regexp" "sort" @@ -265,8 +266,10 @@ func New(options *Options) (*Runner, error) { } func (r *Runner) prepareInputPaths() { + // most likely, the user would provide the most simplified path to an existing file + isAbsoluteOrRelativePath := filepath.Clean(r.options.RequestURIs) == r.options.RequestURIs // Check if the user requested multiple paths - if fileutil.FileExists(r.options.RequestURIs) { + if isAbsoluteOrRelativePath && fileutil.FileExists(r.options.RequestURIs) { r.options.requestURIs = fileutilz.LoadFile(r.options.RequestURIs) } else if r.options.RequestURIs != "" { r.options.requestURIs = strings.Split(r.options.RequestURIs, ",") @@ -917,14 +920,17 @@ retry: } // fix the final output url fullURL := req.URL.String() - parsedURL, _ := urlutil.Parse(fullURL) - if r.options.Unsafe { - parsedURL.RequestURI = reqURI - // if the full url doesn't end with the custom path we pick the original input value - } else if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) { - parsedURL.RequestURI = scanopts.RequestURI - } - fullURL = parsedURL.String() + if parsedURL, errParse := urlutil.Parse(fullURL); errParse != nil { + return Result{URL: URL.String(), Input: origInput, err: errParse} + } else { + if r.options.Unsafe { + parsedURL.RequestURI = reqURI + // if the full url doesn't end with the custom path we pick the original input value + } else if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) { + parsedURL.RequestURI = scanopts.RequestURI + } + fullURL = parsedURL.String() + } if r.options.Debug || r.options.DebugRequests { gologger.Info().Msgf("Dumped HTTP request for %s\n\n", fullURL) From 9286ce243f64cc667ae7b4a56fa8aaf934cfd7ba Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 10 Jan 2022 19:07:06 +0530 Subject: [PATCH 092/355] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 97ac18c..76f43bb 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.5-dev + /_/ v1.1.5 ` // Version is the current version of httpx -const Version = `v1.1.5-dev` +const Version = `v1.1.5` // showBanner is used to show the banner to the user func showBanner() { From c4008cda2c3d0b677fd866fac32ab2c2af319ee2 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 10 Jan 2022 19:35:17 +0530 Subject: [PATCH 093/355] readme updates --- README.md | 77 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d8d103b..54001cc 100644 --- a/README.md +++ b/README.md @@ -49,13 +49,16 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi | Title | true | CNAME | true | | Status Code | true | Raw HTTP | false | | Content Length | true | HTTP2 | false | -| Line Count | true | Word Count | true | -| TLS Certificate | true | HTTP 1.1 Pipeline | false | +| TLS Certificate | true | HTTP Pipeline | false | | CSP Header | true | Virtual host | false | +| Line Count | true | Word Count | true | | Location Header | true | CDN | false | | Web Server | true | Paths | false | | Web Socket | true | Ports | false | -| Response Time | true | Request Method | false | +| Response Time | true | Request Method | true | +| Favicon Hash | false | Probe Status | false | +| Body Hash | true | Header Hash | true | +| Redirect chain | false | URL Scheme | true | # Installation Instructions @@ -189,7 +192,7 @@ OPTIMIZATIONS: # Running httpX -### Running httpx with stdin +### URL Probe This will run the tool against all the hosts and subdomains in `hosts.txt` and returns URLs running HTTP webserver. @@ -218,7 +221,7 @@ https://api.hackerone.com https://support.hackerone.com ``` -### Running httpx with file input +### File Input This will run the tool with the `probe` flag against all of the hosts in **hosts.txt** and return URLs with probed status. @@ -247,7 +250,7 @@ http://a.ns.hackerone.com [FAILED] http://b.ns.hackerone.com [FAILED] ``` -### Running httpx with CIDR input +### CIDR Input ```console echo 173.0.84.0/24 | httpx -silent @@ -273,7 +276,7 @@ https://173.0.84.34 ``` -### Running httpx with subfinder +### Tool Chain ```console @@ -298,7 +301,65 @@ https://support.hackerone.com [301,302,301,200] [HackerOne] [Cloudflare,Ruby on https://resources.hackerone.com [301,301,404] [Sorry, no Folders found.] ``` -### Running httpx with docker +### Favicon Hash + + +```console +subfinder -d hackerone.com -silent | httpx -favicon + + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ v1.1.5 + + projectdiscovery.io + +Use with caution. You are responsible for your actions. +Developers assume no liability and are not responsible for any misuse or damage. +https://docs.hackerone.com/favicon.ico [595148549] +https://hackerone.com/favicon.ico [595148549] +https://mta-sts.managed.hackerone.com/favicon.ico [-1700323260] +https://mta-sts.forwarding.hackerone.com/favicon.ico [-1700323260] +https://support.hackerone.com/favicon.ico [-1279294674] +https://gslink.hackerone.com/favicon.ico [1506877856] +https://resources.hackerone.com/favicon.ico [-1840324437] +https://api.hackerone.com/favicon.ico [566218143] +https://mta-sts.hackerone.com/favicon.ico [-1700323260] +https://www.hackerone.com/favicon.ico [778073381] +``` + +### Path Probe + + +```console +httpx -l urls.txt -path /v1/api -sc + + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ v1.1.5 + + projectdiscovery.io + +Use with caution. You are responsible for your actions. +Developers assume no liability and are not responsible for any misuse or damage. +https://mta-sts.managed.hackerone.com/v1/api [404] +https://mta-sts.hackerone.com/v1/api [404] +https://mta-sts.forwarding.hackerone.com/v1/api [404] +https://docs.hackerone.com/v1/api [404] +https://api.hackerone.com/v1/api [401] +https://hackerone.com/v1/api [302] +https://support.hackerone.com/v1/api [404] +https://resources.hackerone.com/v1/api [301] +https://gslink.hackerone.com/v1/api [404] +http://www.hackerone.com/v1/api [301] +``` + +### Docker Run ```console cat sub_domains.txt | docker run -i projectdiscovery/httpx From 064fbd11ac4270714259cb530f3a819be15e5ad2 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 10 Jan 2022 19:36:47 +0530 Subject: [PATCH 094/355] spacing fixes --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 54001cc..a3b2ed6 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,6 @@

- From ec979b42fa77f7d66231f7d30521a6a95a92abc0 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 10 Jan 2022 19:39:30 +0530 Subject: [PATCH 095/355] dev update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 76f43bb..cd035f1 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.5 + /_/ v1.1.6-dev ` // Version is the current version of httpx -const Version = `v1.1.5` +const Version = `v1.1.6-dev` // showBanner is used to show the banner to the user func showBanner() { From 0fe493f711464179160d51c82ad19061dfcdab78 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 11 Jan 2022 17:32:09 -0600 Subject: [PATCH 096/355] ratelimit per min support --- README.md | 5 +++-- runner/options.go | 2 ++ runner/runner.go | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a3b2ed6..fdacb30 100644 --- a/README.md +++ b/README.md @@ -124,8 +124,9 @@ FILTERS: -ffc, -filter-favicon string[] Filter response with specific favicon RATE-LIMIT: - -t, -threads int Number of threads (default 50) - -rl, -rate-limit int Maximum requests to send per second (default 150) + -t, -threads int Number of threads (default 50) + -rl, -rate-limit int Maximum requests to send per second (default 150) + -rlm, -rate-limit-minute Maximum number of requests to send per minute (default 0) MISCELLANEOUS: -favicon Probes for favicon ("favicon.ico" as path) and display phythonic hash diff --git a/runner/options.go b/runner/options.go index ca2f90e..43cbeb2 100644 --- a/runner/options.go +++ b/runner/options.go @@ -199,6 +199,7 @@ type Options struct { MaxResponseBodySizeToRead int OutputExtractRegex string RateLimit int + RateLimitMinute int Probe bool Resume bool resumeCfg *ResumeCfg @@ -279,6 +280,7 @@ func ParseOptions() *Options { createGroup(flagSet, "rate-limit", "Rate-Limit", flagSet.IntVarP(&options.Threads, "threads", "t", 50, "Number of threads"), flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second"), + flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), ) createGroup(flagSet, "Misc", "Miscellaneous", diff --git a/runner/runner.go b/runner/runner.go index 02814ad..a399e0c 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -249,7 +249,9 @@ func New(options *Options) (*Runner, error) { } runner.hm = hm - if options.RateLimit > 0 { + if options.RateLimitMinute > 0 { + runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second)) + } else if options.RateLimit > 0 { runner.ratelimiter = ratelimit.New(options.RateLimit) } else { runner.ratelimiter = ratelimit.NewUnlimited() From 6c6d3e334f063e0e84d80cccf8dd411eea67e4dd Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sat, 15 Jan 2022 05:50:41 +0530 Subject: [PATCH 097/355] Expose cdnname in output and json --- common/httpx/cdn.go | 4 ++-- go.mod | 2 +- go.sum | 2 ++ runner/runner.go | 8 +++++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/common/httpx/cdn.go b/common/httpx/cdn.go index 7bbf276..073a3e6 100644 --- a/common/httpx/cdn.go +++ b/common/httpx/cdn.go @@ -6,9 +6,9 @@ import ( ) // CdnCheck verifies if the given ip is part of Cdn ranges -func (h *HTTPX) CdnCheck(ip string) (bool, error) { +func (h *HTTPX) CdnCheck(ip string) (bool, string, error) { if h.cdn == nil { - return false, fmt.Errorf("cdn client not configured") + return false, "", fmt.Errorf("cdn client not configured") } return h.cdn.Check(net.ParseIP((ip))) diff --git a/go.mod b/go.mod index 7af0f1d..3d3a296 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.17 github.com/miekg/dns v1.1.43 // indirect github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/cdncheck v0.0.2 + github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e diff --git a/go.sum b/go.sum index 879bcdb..db0be62 100644 --- a/go.sum +++ b/go.sum @@ -105,6 +105,8 @@ github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwa github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= github.com/projectdiscovery/cdncheck v0.0.2 h1:ZAaezMvVioC/8fC8iilylGT4FBBryzQdXHow+lHnPOE= github.com/projectdiscovery/cdncheck v0.0.2/go.mod h1:+vB8400d1Kxq5aDoi/XkVcdvpiqQjSrYJ8bRfUDdymQ= +github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6 h1:pUr8UiK2fPhMmk8qk0ijXH5IaJAtHY47+BODmx62SlU= +github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6/go.mod h1:+vB8400d1Kxq5aDoi/XkVcdvpiqQjSrYJ8bRfUDdymQ= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= diff --git a/runner/runner.go b/runner/runner.go index a399e0c..ae982fd 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1154,9 +1154,9 @@ retry: builder.WriteString(fmt.Sprintf(" [%s]", cnames[0])) } - isCDN, err := hp.CdnCheck(ip) + isCDN, cdnName, err := hp.CdnCheck(ip) if scanopts.OutputCDN && isCDN && err == nil { - builder.WriteString(" [cdn]") + builder.WriteString(fmt.Sprintf(" [%s]", cdnName)) } if scanopts.OutputResponseTime { @@ -1339,6 +1339,7 @@ retry: A: ips, CNAMEs: cnames, CDN: isCDN, + CDNName: cdnName, ResponseTime: resp.Duration.String(), Technologies: technologies, FinalURL: finalURL, @@ -1391,6 +1392,7 @@ type Result struct { Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` HTTP2 bool `json:"http2,omitempty" csv:"http2"` CDN bool `json:"cdn,omitempty" csv:"cdn"` + CDNName string `json:"cdn-name,omitempty" csv:"cdn-name"` ResponseTime string `json:"response-time,omitempty" csv:"response-time"` Technologies []string `json:"technologies,omitempty" csv:"technologies"` Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` @@ -1489,7 +1491,7 @@ func (r *Runner) skipCDNPort(host string, port string) bool { // pick the first ip as target hostIP := dnsData.A[0] - isCdnIP, err := r.hp.CdnCheck(hostIP) + isCdnIP, _, err := r.hp.CdnCheck(hostIP) if err != nil { return false } From 2abd058f83d57c2437812f9562c32edc711fe75d Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sat, 15 Jan 2022 06:13:25 +0530 Subject: [PATCH 098/355] Added stats-interval for changing stats display frequency --- runner/options.go | 2 ++ runner/runner.go | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/runner/options.go b/runner/options.go index 43cbeb2..2a29cee 100644 --- a/runner/options.go +++ b/runner/options.go @@ -191,6 +191,7 @@ type Options struct { TLSGrab bool protocol string ShowStatistics bool + StatsInterval int RandomAgent bool StoreChain bool Deny customlist.CustomList @@ -337,6 +338,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "Show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "Show all received responses"), flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Display scan statistic"), + flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"), ) createGroup(flagSet, "Optimizations", "Optimizations", diff --git a/runner/runner.go b/runner/runner.go index a399e0c..4d7f583 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -54,10 +54,6 @@ import ( "go.uber.org/ratelimit" ) -const ( - statsDisplayInterval = 5 -) - // Runner is a client for running the enumeration process. type Runner struct { options *Options @@ -320,7 +316,7 @@ func (r *Runner) prepareInput() { r.stats.AddCounter("hosts", 0) r.stats.AddStatic("startedAt", time.Now()) r.stats.AddCounter("requests", 0) - err := r.stats.Start(makePrintCallback(), time.Duration(statsDisplayInterval)*time.Second) + err := r.stats.Start(makePrintCallback(), time.Duration(r.options.StatsInterval)*time.Second) if err != nil { gologger.Warning().Msgf("Could not create statistics: %s\n", err) } From 9b9a95a2107f7f082166f4ae60dcba10b4026e36 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sun, 16 Jan 2022 17:34:17 +0530 Subject: [PATCH 099/355] Enable show-statistics if an interval is passed for stats --- runner/options.go | 7 ++++++- runner/runner.go | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 2a29cee..3fda78a 100644 --- a/runner/options.go +++ b/runner/options.go @@ -338,7 +338,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "Show all sent requests"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "Show all received responses"), flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Display scan statistic"), - flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 5, "number of seconds to wait between showing a statistics update"), + flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"), ) createGroup(flagSet, "Optimizations", "Optimizations", @@ -353,6 +353,11 @@ func ParseOptions() *Options { ) _ = flagSet.Parse() + + if options.StatsInterval != 0 { + options.ShowStatistics = true + } + // Read the inputs and configure the logging options.configureOutput() diff --git a/runner/runner.go b/runner/runner.go index 4d7f583..034c655 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -235,6 +235,9 @@ func New(options *Options) (*Runner, error) { if err != nil { return nil, err } + if options.StatsInterval == 0 { + options.StatsInterval = 5 + } } hmapOptions := hybrid.DefaultDiskOptions @@ -316,6 +319,7 @@ func (r *Runner) prepareInput() { r.stats.AddCounter("hosts", 0) r.stats.AddStatic("startedAt", time.Now()) r.stats.AddCounter("requests", 0) + err := r.stats.Start(makePrintCallback(), time.Duration(r.options.StatsInterval)*time.Second) if err != nil { gologger.Warning().Msgf("Could not create statistics: %s\n", err) From dc7a6bf5a8e5383d08e4a1c3890680a580a64734 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jan 2022 10:05:08 +0000 Subject: [PATCH 100/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.24 to 0.0.25. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.24...v0.0.25) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7af0f1d..3e0c809 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.24 + github.com/projectdiscovery/wappalyzergo v0.0.25 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 879bcdb..a70c149 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.24 h1:TQkJKawP+2/30uIG/n04imWtQ5SNt3qLvY7fOf1g0v4= -github.com/projectdiscovery/wappalyzergo v0.0.24/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.25 h1:7C//STQwq0DPExjpXS9EmjxnKb3WWkI1S4MSTog+7+M= +github.com/projectdiscovery/wappalyzergo v0.0.25/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From b160f75224129cee20388e2f3e233a0510678f44 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 19 Jan 2022 23:02:25 -0600 Subject: [PATCH 101/355] stat flag timer --- runner/runner.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 034c655..3ddfc1d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -428,21 +428,14 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { } var ( - lastPrint time.Time lastRequestsCount float64 ) func makePrintCallback() func(stats clistats.StatisticsClient) { builder := &strings.Builder{} return func(stats clistats.StatisticsClient) { - var duration time.Duration - now := time.Now() - if lastPrint.IsZero() { - startedAt, _ := stats.GetStatic("startedAt") - duration = time.Since(startedAt.(time.Time)) - } else { - duration = time.Since(lastPrint) - } + startedAt, _ := stats.GetStatic("startedAt") + duration := time.Since(startedAt.(time.Time)) builder.WriteRune('[') builder.WriteString(clistats.FmtDuration(duration)) @@ -479,7 +472,6 @@ func makePrintCallback() func(stats clistats.StatisticsClient) { fmt.Fprintf(os.Stderr, "%s", builder.String()) builder.Reset() - lastPrint = now lastRequestsCount = currentRequests } } From 80ba8b5ad5334ae68cfcc80932d2047bdcec727d Mon Sep 17 00:00:00 2001 From: Geeknik Labs <466878+geeknik@users.noreply.github.com> Date: Thu, 27 Jan 2022 04:18:28 -0600 Subject: [PATCH 102/355] Update options.go Typo fix --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 3fda78a..911ae7c 100644 --- a/runner/options.go +++ b/runner/options.go @@ -342,7 +342,7 @@ func ParseOptions() *Options { ) createGroup(flagSet, "Optimizations", "Optimizations", - flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "Display both probbed protocol (HTTPS and HTTP)"), + flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "Display both probed protocol (HTTPS and HTTP)"), flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "Probe with input protocol scheme"), flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "Max error count per host before skipping remaining path/s"), flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "Skip full port scans for CDNs (only checks for 80,443)"), From 9ee34f4b1896bc7bfb8e64282f20740bdf1cb551 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 10:06:36 +0000 Subject: [PATCH 103/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.25 to 0.0.27. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.25...v0.0.27) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3e0c809..f4b2c72 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.25 + github.com/projectdiscovery/wappalyzergo v0.0.27 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index a70c149..7ad5198 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.25 h1:7C//STQwq0DPExjpXS9EmjxnKb3WWkI1S4MSTog+7+M= -github.com/projectdiscovery/wappalyzergo v0.0.25/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.27 h1:XvzZJuFWfxR8+cXfsDRKcO1//EeEhvBDmoPpfcloyX4= +github.com/projectdiscovery/wappalyzergo v0.0.27/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From eb3787518f0106f95430a0b5a62e9801c16a613e Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 2 Feb 2022 07:18:57 +0100 Subject: [PATCH 104/355] Adding support for euc-kr charset --- common/httpx/encodings.go | 6 ++++++ common/httpx/title.go | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/common/httpx/encodings.go b/common/httpx/encodings.go index 0c7ba63..f9396a2 100644 --- a/common/httpx/encodings.go +++ b/common/httpx/encodings.go @@ -4,6 +4,7 @@ import ( "bytes" "io/ioutil" + "golang.org/x/text/encoding/korean" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/encoding/traditionalchinese" "golang.org/x/text/transform" @@ -43,3 +44,8 @@ func Encodebig5(s []byte) ([]byte, error) { } return d, nil } + +func DecodeKorean(s []byte) ([]byte, error) { + koreanDecoder := korean.EUCKR.NewDecoder() + return koreanDecoder.Bytes(s) +} diff --git a/common/httpx/title.go b/common/httpx/title.go index ba1c7e3..125f82f 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -7,6 +7,7 @@ import ( "regexp" "strings" + "github.com/projectdiscovery/stringsutil" "golang.org/x/net/html" ) @@ -39,16 +40,21 @@ func ExtractTitle(r *Response) (title string) { // Non UTF-8 if contentTypes, ok := r.Headers["Content-Type"]; ok { - contentType := strings.Join(contentTypes, ";") + contentType := strings.ToLower(strings.Join(contentTypes, ";")) - // special cases - if strings.Contains(strings.ToLower(contentType), "charset=gb2312") || - strings.Contains(strings.ToLower(contentType), "charset=gbk") { + switch { + case stringsutil.ContainsAny(contentType, "charset=gb2312", "charset=gbk"): titleUtf8, err := Decodegbk([]byte(title)) if err != nil { return } + return string(titleUtf8) + case stringsutil.ContainsAny(contentType, "euc-kr"): + titleUtf8, err := DecodeKorean([]byte(title)) + if err != nil { + return + } return string(titleUtf8) } @@ -63,12 +69,12 @@ func ExtractTitle(r *Response) (title string) { } mcontentType = strings.ToLower(mcontentType) } - if strings.Contains(mcontentType, "gb2312") || strings.Contains(mcontentType, "gbk") { + switch { + case stringsutil.ContainsAny(mcontentType, "gb2312", "gbk"): titleUtf8, err := Decodegbk([]byte(title)) if err != nil { return } - return string(titleUtf8) } } From e88fd9594c01214f7a319773dc7db27ef1495d06 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 7 Feb 2022 16:05:05 +0530 Subject: [PATCH 105/355] go mod update --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3d3a296..2001d9b 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.17 github.com/miekg/dns v1.1.43 // indirect github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6 + github.com/projectdiscovery/cdncheck v0.0.3-0.20220207103122-d90a1140381a github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e @@ -52,10 +52,10 @@ require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/json-iterator/go v1.1.11 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect diff --git a/go.sum b/go.sum index db0be62..8b096ce 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= @@ -82,6 +84,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -107,6 +111,8 @@ github.com/projectdiscovery/cdncheck v0.0.2 h1:ZAaezMvVioC/8fC8iilylGT4FBBryzQdX github.com/projectdiscovery/cdncheck v0.0.2/go.mod h1:+vB8400d1Kxq5aDoi/XkVcdvpiqQjSrYJ8bRfUDdymQ= github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6 h1:pUr8UiK2fPhMmk8qk0ijXH5IaJAtHY47+BODmx62SlU= github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6/go.mod h1:+vB8400d1Kxq5aDoi/XkVcdvpiqQjSrYJ8bRfUDdymQ= +github.com/projectdiscovery/cdncheck v0.0.3-0.20220207103122-d90a1140381a h1:U96yKkM0FosBpclr4HHgkeloMKnFexCKPpTt9KEZ1UY= +github.com/projectdiscovery/cdncheck v0.0.3-0.20220207103122-d90a1140381a/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= From d94b8ca232230203e2b26726df89a05a55b53d50 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 7 Feb 2022 19:16:37 +0530 Subject: [PATCH 106/355] go mod update --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2001d9b..16fba50 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.17 github.com/miekg/dns v1.1.43 // indirect github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/cdncheck v0.0.3-0.20220207103122-d90a1140381a + github.com/projectdiscovery/cdncheck v0.0.3 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e diff --git a/go.sum b/go.sum index 8b096ce..8834e45 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,8 @@ github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6 h1:pUr github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6/go.mod h1:+vB8400d1Kxq5aDoi/XkVcdvpiqQjSrYJ8bRfUDdymQ= github.com/projectdiscovery/cdncheck v0.0.3-0.20220207103122-d90a1140381a h1:U96yKkM0FosBpclr4HHgkeloMKnFexCKPpTt9KEZ1UY= github.com/projectdiscovery/cdncheck v0.0.3-0.20220207103122-d90a1140381a/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= +github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g= +github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= From 5a562107f4d0b3aa90a2cbd120b98ca7d3ab6bb4 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 9 Feb 2022 10:09:30 +0100 Subject: [PATCH 107/355] Updating GH workflows + Sonar (#514) * Updating GH workflows + Sonar * adding missing step to build --- .github/dependabot.yml | 10 +++++-- .github/workflows/build-test.yml | 29 ++++++++++++++------ .github/workflows/codeql-analysis.yml | 1 + .github/workflows/functional-test.yml | 8 +++--- .github/workflows/release-binary.yml | 23 +++++++++------- .github/workflows/sonarcloud.yml | 38 +++++++++++++++++++++++++++ cmd/functional-test/run.sh | 26 ++++++++++++------ integration_tests/run.sh | 4 +++ sonar-project.properties | 16 +++++++++++ 9 files changed, 124 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/sonarcloud.yml create mode 100644 sonar-project.properties diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 69d9543..a39e441 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,16 +15,20 @@ updates: commit-message: prefix: "chore" include: "scope" + labels: + - "Type: Maintenance" # Maintain dependencies for go modules - package-ecosystem: "gomod" directory: "/" schedule: - interval: "weekly" + interval: "daily" target-branch: "dev" commit-message: prefix: "chore" include: "scope" + labels: + - "Type: Maintenance" # Maintain dependencies for docker - package-ecosystem: "docker" @@ -34,4 +38,6 @@ updates: target-branch: "dev" commit-message: prefix: "chore" - include: "scope" \ No newline at end of file + include: "scope" + labels: + - "Type: Maintenance" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 3f76a5b..10e0284 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -5,25 +5,38 @@ on: pull_request: workflow_dispatch: - jobs: build: name: Test Builds - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-go@v2 + - name: Set up Go + uses: actions/setup-go@v2 with: go-version: 1.17 + - name: Check out code + uses: actions/checkout@v2 + + - name: Build + run: go build . + working-directory: cmd/httpx/ + - name: Test run: go test ./... + working-directory: . - name: Integration Tests + env: + GH_ACTION: true run: bash run.sh working-directory: integration_tests/ - - - name: Build - run: go build . + + - name: Race Condition Tests + run: go build -race . working-directory: cmd/httpx/ + + diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 545cdea..792785b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,6 +2,7 @@ name: 🚨 CodeQL Analysis on: workflow_dispatch: + push: pull_request: branches: - dev diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 027f3c7..9d48a45 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -4,11 +4,13 @@ on: pull_request: workflow_dispatch: - jobs: functional: name: Functional Test - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go uses: actions/setup-go@v2 @@ -21,5 +23,5 @@ jobs: - name: Functional Tests run: | chmod +x run.sh - bash run.sh + bash run.sh ${{ matrix.os }} working-directory: cmd/functional-test diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 7cf8f99..657340d 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -9,18 +9,21 @@ jobs: release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-go@v2 - with: + - name: "Check out code" + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: "Set up Go" + uses: actions/setup-go@v2 + with: go-version: 1.17 - - - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "Create release on GitHub" + + - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v2 - with: + with: args: "release --rm-dist" version: latest workdir: . + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml new file mode 100644 index 0000000..62ad03f --- /dev/null +++ b/.github/workflows/sonarcloud.yml @@ -0,0 +1,38 @@ +name: 👮🏼‍♂️ Sonarcloud +on: + push: + branches: + - master + - dev + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +jobs: + sonarcloud: + name: SonarCloud + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + + - name: "Set up Go" + uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Run unit Tests + run: | + go test -coverprofile=./cov.out ./... + + - name: Run Gosec Security Scanner + run: | + go install github.com/securego/gosec/cmd/gosec@latest + gosec -no-fail -fmt=sonarqube -out report.json ./... + + - name: SonarCloud Scan + uses: SonarSource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file diff --git a/cmd/functional-test/run.sh b/cmd/functional-test/run.sh index be5e8c7..2d4ddb8 100755 --- a/cmd/functional-test/run.sh +++ b/cmd/functional-test/run.sh @@ -1,13 +1,23 @@ #!/bin/bash -echo 'Building functional-test binary' -go build +# reading os type from arguments +CURRENT_OS=$1 -echo 'Building HTTPX binary from current branch' -go build -o httpx_dev ../httpx +if [ "${CURRENT_OS}" == "windows-latest" ];then + extension=.exe +fi -echo 'Installing latest release of HTTPX' -GO111MODULE=on go build -v github.com/projectdiscovery/httpx/cmd/httpx +echo "::group::Building functional-test binary" +go build -o functional-test$extension +echo "::endgroup::" -echo 'Starting HTTPX functional test' -./functional-test -main ./httpx -dev ./httpx_dev -testcases testcases.txt +echo "::group::Building dnsx binary from current branch" +go build -o httpx_dev$extension ../httpx +echo "::endgroup::" + +echo "::group::Building latest release of dnsx" +go build -o httpx$extension -v github.com/projectdiscovery/httpx/cmd/httpx +echo "::endgroup::" + +echo 'Starting dnsx functional test' +./functional-test$extension -main ./httpx$extension -dev ./httpx_dev$extension -testcases testcases.txt diff --git a/integration_tests/run.sh b/integration_tests/run.sh index cb9cbf3..ab8a110 100755 --- a/integration_tests/run.sh +++ b/integration_tests/run.sh @@ -1,13 +1,17 @@ #!/bin/bash +echo "::group::Build httpx" rm integration-test httpx 2>/dev/null cd ../cmd/httpx go build mv httpx ../../integration_tests/httpx +echo "::endgroup::" +echo "::group::Build httpx integration-test" cd ../integration-test go build mv integration-test ../../integration_tests/integration-test cd ../../integration_tests +echo "::endgroup::" ./integration-test if [ $? -eq 0 ] then diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..1b99918 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,16 @@ +sonar.projectKey=projectdiscovery_httpx +sonar.organization=projectdiscovery + +# This is the name and version displayed in the SonarCloud UI. +#sonar.projectName=dnsx +#sonar.projectVersion=1.0 + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +sonar.sources=. +sonar.tests=. +sonar.test.inclusions=**/*_test.go +sonar.go.coverage.reportPaths=cov.out +sonar.externalIssuesReportPaths=report.json + +# Encoding of the source code. Default is default system encoding +#sonar.sourceEncoding=UTF-8 \ No newline at end of file From 687ce8a44cb7b524a1e08c7ee914250c4a5c529a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 15:41:06 +0530 Subject: [PATCH 108/355] chore(deps): bump github.com/microcosm-cc/bluemonday (#520) Bumps [github.com/microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday) from 1.0.17 to 1.0.18. - [Release notes](https://github.com/microcosm-cc/bluemonday/releases) - [Commits](https://github.com/microcosm-cc/bluemonday/compare/v1.0.17...v1.0.18) --- updated-dependencies: - dependency-name: github.com/microcosm-cc/bluemonday dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 932f74e..16947b5 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/microcosm-cc/bluemonday v1.0.17 + github.com/microcosm-cc/bluemonday v1.0.18 github.com/miekg/dns v1.1.43 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.3 diff --git a/go.sum b/go.sum index 3a3b382..79e1a9a 100644 --- a/go.sum +++ b/go.sum @@ -51,7 +51,6 @@ github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6 github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -72,8 +71,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/microcosm-cc/bluemonday v1.0.17 h1:Z1a//hgsQ4yjC+8zEkV8IWySkXnsxmdSY642CTFQb5Y= -github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= +github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= @@ -82,7 +81,6 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= @@ -107,12 +105,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= -github.com/projectdiscovery/cdncheck v0.0.2 h1:ZAaezMvVioC/8fC8iilylGT4FBBryzQdXHow+lHnPOE= -github.com/projectdiscovery/cdncheck v0.0.2/go.mod h1:+vB8400d1Kxq5aDoi/XkVcdvpiqQjSrYJ8bRfUDdymQ= -github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6 h1:pUr8UiK2fPhMmk8qk0ijXH5IaJAtHY47+BODmx62SlU= -github.com/projectdiscovery/cdncheck v0.0.3-0.20220115001641-2550ce07bde6/go.mod h1:+vB8400d1Kxq5aDoi/XkVcdvpiqQjSrYJ8bRfUDdymQ= -github.com/projectdiscovery/cdncheck v0.0.3-0.20220207103122-d90a1140381a h1:U96yKkM0FosBpclr4HHgkeloMKnFexCKPpTt9KEZ1UY= -github.com/projectdiscovery/cdncheck v0.0.3-0.20220207103122-d90a1140381a/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g= github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= From 4a6672a74286d310041ddc1179e39da6cd73518f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 15:41:18 +0530 Subject: [PATCH 109/355] chore(deps): bump golang from 1.17.6-alpine to 1.17.7-alpine (#518) Bumps golang from 1.17.6-alpine to 1.17.7-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e18ce75..5dae619 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.6-alpine AS builder +FROM golang:1.17.7-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 8865959f8ef32ae4f2216eafeb700bc9c08c294b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 15:41:33 +0530 Subject: [PATCH 110/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#519) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.27 to 0.0.28. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.27...v0.0.28) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 16947b5..96a22a5 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.27 + github.com/projectdiscovery/wappalyzergo v0.0.28 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 79e1a9a..bfb74c7 100644 --- a/go.sum +++ b/go.sum @@ -158,8 +158,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.27 h1:XvzZJuFWfxR8+cXfsDRKcO1//EeEhvBDmoPpfcloyX4= -github.com/projectdiscovery/wappalyzergo v0.0.27/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.28 h1:yYqMxPjYjkYnKHdKeHXOwqR0WAhHGOuzmXYnjd8xX8M= +github.com/projectdiscovery/wappalyzergo v0.0.28/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From e2b0c1683bed37698a8866efa2c6fe99b0ed8199 Mon Sep 17 00:00:00 2001 From: M09Ic Date: Tue, 15 Feb 2022 20:02:33 +0800 Subject: [PATCH 111/355] Add hash flag, to support multi body hash type. (#517) * Add hashes flag, to support multi body hash type. issue:489 and issue:488 * go mod update * Add hash type validate for -hash flag Co-authored-by: sandeep Co-authored-by: mzack --- common/hashes/doc.go | 1 + common/hashes/hashes.go | 64 +++++++++++++++++++++++++++++++++++++++++ common/slice/slice.go | 10 +++++++ go.mod | 2 ++ go.sum | 2 ++ runner/options.go | 13 +++++++++ runner/runner.go | 36 +++++++++++++++++++++++ 7 files changed, 128 insertions(+) create mode 100644 common/hashes/doc.go create mode 100644 common/hashes/hashes.go diff --git a/common/hashes/doc.go b/common/hashes/doc.go new file mode 100644 index 0000000..0067bdd --- /dev/null +++ b/common/hashes/doc.go @@ -0,0 +1 @@ +package hashes diff --git a/common/hashes/hashes.go b/common/hashes/hashes.go new file mode 100644 index 0000000..a152877 --- /dev/null +++ b/common/hashes/hashes.go @@ -0,0 +1,64 @@ +package hashes + +import ( + "bytes" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "fmt" + "github.com/mfonda/simhash" + "github.com/spaolacci/murmur3" +) + +func stdBase64(braw []byte) []byte { + bckd := base64.StdEncoding.EncodeToString(braw) + var buffer bytes.Buffer + for i := 0; i < len(bckd); i++ { + ch := bckd[i] + buffer.WriteByte(ch) + if (i+1)%76 == 0 { + buffer.WriteByte('\n') + } + } + buffer.WriteByte('\n') + return buffer.Bytes() +} + +func Mmh3(data []byte) string { + var h32 = murmur3.New32WithSeed(0) + h32.Write(stdBase64(data)) + return fmt.Sprintf("%d", h32.Sum32()) +} + +func Md5(data []byte) string { + hash := md5.Sum(data) + return hex.EncodeToString(hash[:]) +} + +func Sha1(data []byte) string { + hash := sha1.Sum(data) + return hex.EncodeToString(hash[:]) +} + +func Sha256(data []byte) string { + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} + +func Sha224(data []byte) string { + hash := sha256.Sum224(data) + return hex.EncodeToString(hash[:]) +} + +func Sha512(data []byte) string { + hash := sha512.Sum512(data) + return hex.EncodeToString(hash[:]) +} + +func Simhash(data []byte) string { + hash := simhash.Simhash(simhash.NewWordFeatureSet(data)) + return fmt.Sprintf("%d", hash) +} diff --git a/common/slice/slice.go b/common/slice/slice.go index 2426d84..ce837b9 100644 --- a/common/slice/slice.go +++ b/common/slice/slice.go @@ -20,6 +20,16 @@ func UInt32SliceContains(sl []uint32, v uint32) bool { return false } +// StringSliceContains check if a slice contains the specified int value +func StringSliceContains(sl []string, v string) bool { + for _, vv := range sl { + if vv == v { + return true + } + } + return false +} + // ToSlice creates a slice with all string keys from a map func ToSlice(m map[string]struct{}) (s []string) { for k := range m { diff --git a/go.mod b/go.mod index 96a22a5..6b92331 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,8 @@ require ( require github.com/spaolacci/murmur3 v1.1.0 +require github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 + require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index bfb74c7..5479a83 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= +github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= diff --git a/runner/options.go b/runner/options.go index 911ae7c..5e3d01b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -1,6 +1,7 @@ package runner import ( + "github.com/projectdiscovery/httpx/common/slice" "math" "os" "regexp" @@ -71,6 +72,7 @@ type scanOptions struct { LeaveDefaultPorts bool OutputLinesCount bool OutputWordsCount bool + Hashes string } func (s *scanOptions) Clone() *scanOptions { @@ -114,6 +116,7 @@ func (s *scanOptions) Clone() *scanOptions { LeaveDefaultPorts: s.LeaveDefaultPorts, OutputLinesCount: s.OutputLinesCount, OutputWordsCount: s.OutputWordsCount, + Hashes: s.Hashes, } } @@ -224,6 +227,7 @@ type Options struct { matchWordsCount []int OutputFilterWordsCount string filterWordsCount []int + Hashes string } // ParseOptions parses the command line options for application @@ -295,6 +299,7 @@ func ParseOptions() *Options { flagSet.VarP(&options.CustomPorts, "ports", "p", "Port to scan (nmap syntax: eg 1,2-10,11)"), flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"), flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"), + flagSet.StringVar(&options.Hashes, "hash", "", "Probes for body multi hashes"), ) createGroup(flagSet, "output", "Output", @@ -460,6 +465,14 @@ func (options *Options) validateOptions() { gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n") options.RequestURIs = "/favicon.ico" } + + if options.Hashes != "" { + for _, hashType := range strings.Split(options.Hashes, ",") { + if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) { + gologger.Error().Msgf("Unsupported hash type: %s\n", hashType) + } + } + } } // configureOutput configures the output on the screen diff --git a/runner/runner.go b/runner/runner.go index 8484ef1..0f28da8 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -30,6 +30,7 @@ import ( "github.com/projectdiscovery/clistats" "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/goconfig" + "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" @@ -228,6 +229,7 @@ func New(options *Options) (*Runner, error) { scanopts.LeaveDefaultPorts = options.LeaveDefaultPorts scanopts.OutputLinesCount = options.OutputLinesCount scanopts.OutputWordsCount = options.OutputWordsCount + scanopts.Hashes = options.Hashes runner.scanopts = scanopts if options.ShowStatistics { @@ -1211,6 +1213,38 @@ retry: builder.WriteRune(']') } + var hashesMap = map[string]string{} + if scanopts.Hashes != "" { + hs := strings.Split(scanopts.Hashes, ",") + for _, hashType := range hs { + var hash string + switch strings.ToLower(hashType) { + case "md5": + hash = hashes.Md5(resp.Data) + case "mmh3": + hash = hashes.Mmh3(resp.Data) + case "sha1": + hash = hashes.Sha1(resp.Data) + case "sha256": + hash = hashes.Sha256(resp.Data) + case "sha512": + hash = hashes.Sha512(resp.Data) + case "simhash": + hash = hashes.Simhash(resp.Data) + } + if hash != "" { + hashesMap[hashType] = hash + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(hash).String()) + } else { + builder.WriteString(hash) + } + builder.WriteRune(']') + } + } + } + if scanopts.OutputLinesCount { builder.WriteString(" [") if !scanopts.OutputWithNoColor { @@ -1336,6 +1370,7 @@ retry: Technologies: technologies, FinalURL: finalURL, FavIconMMH3: faviconMMH3, + Hashes: hashesMap, Lines: resp.Lines, Words: resp.Words, } @@ -1391,6 +1426,7 @@ type Result struct { FinalURL string `json:"final-url,omitempty" csv:"final-url"` Failed bool `json:"failed" csv:"failed"` FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` + Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` Lines int `json:"lines" csv:"lines"` Words int `json:"words" csv:"words"` } From 22ab14ae30959442288ce4b77cf3e761cff4e72f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Feb 2022 16:19:27 +0530 Subject: [PATCH 112/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#524) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.28 to 0.0.29. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.28...v0.0.29) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6b92331..e22fa23 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.28 + github.com/projectdiscovery/wappalyzergo v0.0.29 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 5479a83..493dd9e 100644 --- a/go.sum +++ b/go.sum @@ -160,8 +160,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.28 h1:yYqMxPjYjkYnKHdKeHXOwqR0WAhHGOuzmXYnjd8xX8M= -github.com/projectdiscovery/wappalyzergo v0.0.28/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.29 h1:rpFD6LYY0Us3nlAJlA/QCoaLFyyrfuwhXYVTKDAA5x4= +github.com/projectdiscovery/wappalyzergo v0.0.29/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 9dd6f206e2ed2ddf7443bbd19485fec76b24cb80 Mon Sep 17 00:00:00 2001 From: Sajad Date: Thu, 24 Feb 2022 13:26:08 +0530 Subject: [PATCH 113/355] add env MSYS_NO_PATHCONV to windows build test to prevent path expansion (#526) --- .github/workflows/build-test.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 10e0284..884a579 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -29,14 +29,21 @@ jobs: run: go test ./... working-directory: . - - name: Integration Tests + - name: Integration Tests Linux, macOS + if: runner.os == 'Linux' || runner.os == 'macOS' env: GH_ACTION: true run: bash run.sh working-directory: integration_tests/ + + - name: Integration Tests Windows + if: runner.os == 'Windows' + env: + GH_ACTION: true + MSYS_NO_PATHCONV: true + run: bash run.sh + working-directory: integration_tests/ - name: Race Condition Tests run: go build -race . working-directory: cmd/httpx/ - - From ef22a65b55b60d63b27e9706b98d134eaa46469b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 16:33:31 +0530 Subject: [PATCH 114/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#531) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.29 to 0.0.30. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.29...v0.0.30) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e22fa23..f62264d 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.29 + github.com/projectdiscovery/wappalyzergo v0.0.30 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 493dd9e..6908260 100644 --- a/go.sum +++ b/go.sum @@ -160,8 +160,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.29 h1:rpFD6LYY0Us3nlAJlA/QCoaLFyyrfuwhXYVTKDAA5x4= -github.com/projectdiscovery/wappalyzergo v0.0.29/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.30 h1:DngC7onROGD+ugfo1xV3iRzi8l+dlbO7RxQ5ojZ2zuY= +github.com/projectdiscovery/wappalyzergo v0.0.30/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 20c5b608dfcd5a27b061b8fca88e3fed8149aa06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 16:34:05 +0530 Subject: [PATCH 115/355] chore(deps): bump golangci/golangci-lint-action from 2 to 3.1.0 (#530) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 2 to 3.1.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v2...v3.1.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 794d073..5fbc57f 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -12,7 +12,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3.1.0 with: version: latest args: --timeout 5m From 03ded973d8d3be26cfbd149cb5740206fafa19ca Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Tue, 1 Mar 2022 00:35:06 -0600 Subject: [PATCH 116/355] hash improvement (#528) * hash improvement * mmh3 conversion to int32 * misc update Co-authored-by: sandeep --- README.md | 1 + common/hashes/hashes.go | 2 +- runner/options.go | 2 +- runner/runner.go | 64 ++++++++++++++++++++--------------------- 4 files changed, 35 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index fdacb30..447778d 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ MISCELLANEOUS: -p, -ports string[] Port to scan (nmap syntax: eg 1,2-10,11) -path string File or comma separated paths to request -paths string File or comma separated paths to request (deprecated) + -hash string Display response body hash (supported: md5,mmh3,sha256,sim) OUTPUT: -o, -output string file to write output diff --git a/common/hashes/hashes.go b/common/hashes/hashes.go index a152877..884968d 100644 --- a/common/hashes/hashes.go +++ b/common/hashes/hashes.go @@ -30,7 +30,7 @@ func stdBase64(braw []byte) []byte { func Mmh3(data []byte) string { var h32 = murmur3.New32WithSeed(0) h32.Write(stdBase64(data)) - return fmt.Sprintf("%d", h32.Sum32()) + return fmt.Sprintf("%d", int32(h32.Sum32())) } func Md5(data []byte) string { diff --git a/runner/options.go b/runner/options.go index 5e3d01b..97268cc 100644 --- a/runner/options.go +++ b/runner/options.go @@ -299,7 +299,7 @@ func ParseOptions() *Options { flagSet.VarP(&options.CustomPorts, "ports", "p", "Port to scan (nmap syntax: eg 1,2-10,11)"), flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"), flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"), - flagSet.StringVar(&options.Hashes, "hash", "", "Probes for body multi hashes"), + flagSet.StringVar(&options.Hashes, "hash", "", "Display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"), ) createGroup(flagSet, "output", "Output", diff --git a/runner/runner.go b/runner/runner.go index 0f28da8..c7bcf0a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -4,9 +4,7 @@ import ( "bufio" "bytes" "context" - "crypto/sha256" "encoding/csv" - "encoding/hex" "encoding/json" "fmt" "io/ioutil" @@ -1212,39 +1210,54 @@ retry: } builder.WriteRune(']') } - + // adding default hashing for json output format + if r.options.JSONOutput && len(scanopts.Hashes) == 0 { + scanopts.Hashes = "md5,mmh3,sha256,simhash" + } var hashesMap = map[string]string{} if scanopts.Hashes != "" { hs := strings.Split(scanopts.Hashes, ",") - for _, hashType := range hs { - var hash string - switch strings.ToLower(hashType) { + builder.WriteString(" [") + for index, hashType := range hs { + var ( + hashHeader, hashBody string + ) + hashType = strings.ToLower(hashType) + switch hashType { case "md5": - hash = hashes.Md5(resp.Data) + hashBody = hashes.Md5(resp.Data) + hashHeader = hashes.Md5([]byte(resp.RawHeaders)) case "mmh3": - hash = hashes.Mmh3(resp.Data) + hashBody = hashes.Mmh3(resp.Data) + hashHeader = hashes.Mmh3([]byte(resp.RawHeaders)) case "sha1": - hash = hashes.Sha1(resp.Data) + hashBody = hashes.Sha1(resp.Data) + hashHeader = hashes.Sha1([]byte(resp.RawHeaders)) case "sha256": - hash = hashes.Sha256(resp.Data) + hashBody = hashes.Sha256(resp.Data) + hashHeader = hashes.Sha256([]byte(resp.RawHeaders)) case "sha512": - hash = hashes.Sha512(resp.Data) + hashBody = hashes.Sha512(resp.Data) + hashHeader = hashes.Sha512([]byte(resp.RawHeaders)) case "simhash": - hash = hashes.Simhash(resp.Data) + hashBody = hashes.Simhash(resp.Data) + hashHeader = hashes.Simhash([]byte(resp.RawHeaders)) } - if hash != "" { - hashesMap[hashType] = hash - builder.WriteString(" [") + if hashBody != "" { + hashesMap[fmt.Sprintf("body-%s", hashType)] = hashBody + hashesMap[fmt.Sprintf("header-%s", hashType)] = hashHeader if !scanopts.OutputWithNoColor { - builder.WriteString(aurora.Magenta(hash).String()) + builder.WriteString(aurora.Magenta(hashBody).String()) } else { - builder.WriteString(hash) + builder.WriteString(hashBody) + } + if index != len(hs)-1 { + builder.WriteString(",") } - builder.WriteRune(']') } } + builder.WriteRune(']') } - if scanopts.OutputLinesCount { builder.WriteString(" [") if !scanopts.OutputWithNoColor { @@ -1314,15 +1327,6 @@ retry: if finalPath == "" { finalPath = "/" } - - hasher := sha256.New() - _, _ = hasher.Write(resp.Data) - bodySha := hex.EncodeToString(hasher.Sum(nil)) - hasher.Reset() - - _, _ = hasher.Write([]byte(resp.RawHeaders)) - headersSha := hex.EncodeToString(hasher.Sum(nil)) - var chainStatusCodes []int if resp.HasChain() { chainStatusCodes = append(chainStatusCodes, resp.GetChainStatusCodes()...) @@ -1339,8 +1343,6 @@ retry: Scheme: parsed.Scheme, Port: finalPort, Path: finalPath, - BodySHA256: bodySha, - HeaderSHA256: headersSha, raw: resp.Raw, URL: fullURL, Input: origInput, @@ -1392,8 +1394,6 @@ type Result struct { Scheme string `json:"scheme,omitempty" csv:"scheme"` Port string `json:"port,omitempty" csv:"port"` Path string `json:"path,omitempty" csv:"path"` - BodySHA256 string `json:"body-sha256,omitempty" csv:"body-sha256"` - HeaderSHA256 string `json:"header-sha256,omitempty" csv:"header-sha256"` A []string `json:"a,omitempty" csv:"a"` CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` raw string From 8e7f8952fccf02253baa7f877b7776b714948767 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 1 Mar 2022 14:28:37 +0530 Subject: [PATCH 117/355] readme and banner update --- README.md | 168 ++++++++++++++++++++++++---------------------- runner/banner.go | 4 +- runner/options.go | 163 ++++++++++++++++++++++---------------------- 3 files changed, 171 insertions(+), 164 deletions(-) diff --git a/README.md b/README.md index 447778d..c9d707e 100644 --- a/README.md +++ b/README.md @@ -83,112 +83,116 @@ Usage: Flags: INPUT: - -l, -list string Input file containing list of hosts to process - -request string File containing raw request + -l, -list string input file containing list of hosts to process + -rr, -request string file containing raw request PROBES: - -sc, -status-code Display Status Code - -td, -tech-detect Display wappalyzer based technology detection - -cl, -content-length Display Content-Length - -lc, -line-count Display Response body line count - -wc, -word-count Display Response body word count - -server, -web-server Display Server header - -ct, -content-type Display Content-Type header - -rt, -response-time Display the response time - -title Display page title - -location Display Location header - -method Display Request method - -websocket Display server using websocket - -ip Display Host IP - -cname Display Host cname - -cdn Display if CDN in use - -probe Display probe status + -sc, -status-code display response status-code + -cl, -content-length display response content-length + -ct, -content-type display response content-type + -location display response redirect location + -favicon display mmh3 hash for '/favicon.ico' file + -hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512) + -rt, -response-time display response time + -lc, -line-count display response body line count + -wc, -word-count display response body word count + -title display page title + -server, -web-server display server name + -td, -tech-detect display technology in use based on wappalyzer dataset + -method display http request method + -websocket display server using websocket + -ip display host ip + -cname display host cname + -cdn display cdn in use + -probe display probe status MATCHERS: - -mc, -match-code string Match response with given status code (-mc 200,302) - -ml, -match-length string Match response with given content length (-ml 100,102) - -ms, -match-string string Match response with given string - -mr, -match-regex string Match response with specific regex - -er, -extract-regex string Display response content with matched regex - -mlc, -match-line-count string Match Response body line count - -mwc, -match-word-count string Match Response body word count - -mfc, -match-favicon string[] Match response with specific favicon + -mc, -match-code string match response with specified status code (-mc 200,302) + -ml, -match-length string match response with specified content length (-ml 100,102) + -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) + -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) + -ms, -match-string string match response with specified string (-ms admin) + -mr, -match-regex string match response with specified regex (-mr admin) + -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) + +EXTRACTOR: + -er, -extract-regex string display response content for specified regex FILTERS: - -fc, -filter-code string Filter response with given status code (-fc 403,401) - -fl, -filter-length string Filter response with given content length (-fl 23,33) - -fs, -filter-string string Filter response with specific string - -fe, -filter-regex string Filter response with specific regex - -flc, -filter-line-count string Filter Response body line count - -fwc, -filter-word-count string Filter Response body word count - -ffc, -filter-favicon string[] Filter response with specific favicon + -fc, -filter-code string filter response with specified status code (-fc 403,401) + -fl, -filter-length string filter response with specified content length (-fl 23,33) + -fs, -filter-string string filter response with specified string (-fs admin) + -fe, -filter-regex string filter response with specified regex (-fe admin) + -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) + -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) + -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) RATE-LIMIT: - -t, -threads int Number of threads (default 50) - -rl, -rate-limit int Maximum requests to send per second (default 150) - -rlm, -rate-limit-minute Maximum number of requests to send per minute (default 0) + -t, -threads int number of threads to use (default 50) + -rl, -rate-limit int maximum requests to send per second (default 150) + -rlm, -rate-limit-minute int maximum number of requests to send per minute MISCELLANEOUS: - -favicon Probes for favicon ("favicon.ico" as path) and display phythonic hash - -tls-grab Perform TLS(SSL) data grabbing - -tls-probe Send HTTP probes on the extracted TLS domains - -csp-probe Send HTTP probes on the extracted CSP domains - -pipeline HTTP1.1 Pipeline probe - -http2 HTTP2 probe - -vhost VHOST Probe - -p, -ports string[] Port to scan (nmap syntax: eg 1,2-10,11) - -path string File or comma separated paths to request - -paths string File or comma separated paths to request (deprecated) - -hash string Display response body hash (supported: md5,mmh3,sha256,sim) + -pa, -probe-all-ips probe all the ips associated with same host + -tls-grab perform TLS(SSL) data grabbing + -tls-probe send http probes on the extracted TLS domains (dns_name) + -csp-probe send http probes on the extracted CSP domains + -pipeline probe and display server supporting HTTP1.1 pipeline + -http2 probe and display server supporting HTTP2 + -vhost probe and display server supporting VHOST + -p, -ports string[] ports to probe (nmap syntax: eg 1,2-10,11) + -path string path or list of paths to probe (comma-separated, file) + -paths string path or list of paths to probe (comma-separated, file) (deprecated) OUTPUT: - -o, -output string file to write output + -o, -output string file to write output results -sr, -store-response store http response to output directory - -srd, -store-response-dir string store http response to custom directory (default "output") - -csv store output in CSV format + -srd, -store-response-dir string store http response to custom directory + -csv store output in csv format -json store output in JSONL(ines) format -irr, -include-response include http request/response in JSON output (-json only) -include-chain include redirect http chain in JSON output (-json only) -store-chain include http redirect chain in responses (-sr only) CONFIGURATIONS: - -r, -resolvers string[] List of custom resolvers (file or comma separated) - -allow string[] Allowed list of IP/CIDR's to process (file or comma separated) - -deny string[] Denied list of IP/CIDR's to process (file or comma separated) + -r, -resolvers string[] list of custom resolvers (file or comma separated) + -allow string[] allowed list of IP/CIDR's to process (file or comma separated) + -deny string[] denied list of IP/CIDR's to process (file or comma separated) -random-agent Enable Random User-Agent to use (default true) - -H, -header string[] Custom Header to send with request - -http-proxy, -proxy string HTTP Proxy, eg http://127.0.0.1:8080 - -unsafe Send raw requests skipping golang normalization - -resume Resume scan using resume.cfg - -fr, -follow-redirects Follow HTTP redirects - -maxr, -max-redirects int Max number of redirects to follow per host (default 10) - -fhr, -follow-host-redirects Follow redirects on the same host - -vhost-input Get a list of vhosts as input - -x string Request methods to use, use 'all' to probe all HTTP methods - -body string Post body to include in HTTP request - -s, -stream Stream mode - start elaborating input targets without sorting - -sd, -skip-dedupe Disable dedupe input items (only used with stream mode) - -pa, -probe-all-ips Probe all the ips associated with same host + -H, -header string[] custom http headers to send with request + -http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080) + -unsafe send raw requests skipping golang normalization + -resume resume scan using resume.cfg + -fr, -follow-redirects follow http redirects + -maxr, -max-redirects int max number of redirects to follow per host (default 10) + -fhr, -follow-host-redirects follow redirects on the same host + -vhost-input get a list of vhosts as input + -x string request methods to probe, use 'all' to probe all HTTP methods + -body string post body to include in http request + -s, -stream stream mode - start elaborating input targets without sorting + -sd, -skip-dedupe disable dedupe input items (only used with stream mode) + -ldp, -leave-default-ports leave default http/https ports (eg. http://host:80 - https//host:443 DEBUG: - -silent Silent mode - -v, -verbose Verbose mode - -version Display version - -nc, -no-color Disable color in output - -debug Debug mode - -debug-req Show all sent requests - -debug-resp Show all received responses - -stats Display scan statistic + -debug display request/response content in cli + -debug-req display request content in cli + -debug-resp display response content in cli + -version display httpx version + -stats display scan statistic + -silent silent mode + -v, -verbose verbose mode + -si, -stats-interval int number of seconds to wait between showing a statistics update (default: 5) + -nc, -no-color disable colors in cli output OPTIMIZATIONS: - -nf, -no-fallback Display both probbed protocol (HTTPS and HTTP) - -nfs, -no-fallback-scheme Probe with input protocol scheme - -maxhr, -max-host-error int Max error count per host before skipping remaining path/s (default 30) - -ec, -exclude-cdn Skip full port scans for CDNs (only checks for 80,443) - -retries int Number of retries - -timeout int Timeout in seconds (default 5) - -rsts, -response-size-to-save int Max response size to save in bytes (default 2147483647) - -rstr, -response-size-to-read int Max response size to read in bytes (default 2147483647) + -nf, -no-fallback display both probed protocol (HTTPS and HTTP) + -nfs, -no-fallback-scheme probe with protocol scheme specified in input + -maxhr, -max-host-error int max error count per host before skipping remaining path/s (default 30) + -ec, -exclude-cdn skip full port scans for CDNs (only checks for 80,443) + -retries int number of retries + -timeout int timeout in seconds (default 5) + -rsts, -response-size-to-save int max response size to save in bytes (default 2147483647) + -rstr, -response-size-to-read int max response size to read in bytes (default 2147483647) ``` # Running httpX diff --git a/runner/banner.go b/runner/banner.go index cd035f1..1608a2f 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.6-dev + /_/ v1.1.6 ` // Version is the current version of httpx -const Version = `v1.1.6-dev` +const Version = `v1.1.6` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/options.go b/runner/options.go index 97268cc..7fac914 100644 --- a/runner/options.go +++ b/runner/options.go @@ -238,75 +238,79 @@ func ParseOptions() *Options { flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library.`) createGroup(flagSet, "input", "Input", - flagSet.StringVarP(&options.InputFile, "list", "l", "", "Input file containing list of hosts to process"), - flagSet.StringVar(&options.InputRawRequest, "request", "", "File containing raw request"), + flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"), + flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"), ) createGroup(flagSet, "Probes", "Probes", - flagSet.BoolVarP(&options.StatusCode, "status-code", "sc", false, "Display Status Code"), - flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "Display wappalyzer based technology detection"), - flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "Display Content-Length"), - flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "Display Server header"), - flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "Display Content-Type header"), - flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "Display Response body line count"), - flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "Display Response body word count"), - flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "Display the response time"), - flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"), - flagSet.BoolVar(&options.Location, "location", false, "Display Location header"), - flagSet.BoolVar(&options.OutputMethod, "method", false, "Display Request method"), - flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "Display server using websocket"), - flagSet.BoolVar(&options.OutputIP, "ip", false, "Display Host IP"), - flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"), - flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"), - flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"), + flagSet.BoolVarP(&options.StatusCode, "status-code", "sc", false, "display response status-code"), + flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "display response content-length"), + flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "display response content-type"), + flagSet.BoolVar(&options.Location, "location", false, "display response redirect location"), + flagSet.BoolVar(&options.Favicon, "favicon", false, "display mmh3 hash for '/favicon.ico' file"), + flagSet.StringVar(&options.Hashes, "hash", "", "display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"), + flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "display response time"), + flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "display response body line count"), + flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "display response body word count"), + flagSet.BoolVar(&options.ExtractTitle, "title", false, "display page title"), + flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"), + flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"), + flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"), + flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"), + flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"), + flagSet.BoolVar(&options.OutputCName, "cname", false, "display host cname"), + flagSet.BoolVar(&options.OutputCDN, "cdn", false, "display cdn in use"), + flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), ) createGroup(flagSet, "matchers", "Matchers", - flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "Match response with given status code (-mc 200,302)"), - flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"), - flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"), - flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"), - flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"), - flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"), - flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"), - flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "Match response with specific favicon"), + flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "match response with specified status code (-mc 200,302)"), + flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"), + flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"), + flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"), + flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), + flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), + flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "match response with specified favicon hash (-mfc 1494302000)"), + ) + + createGroup(flagSet, "extractor", "Extractor", + flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "display response content for specified regex"), ) createGroup(flagSet, "filters", "Filters", - flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "Filter response with given status code (-fc 403,401)"), - flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"), - flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"), - flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"), - flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"), - flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"), - flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "Filter response with specific favicon"), + flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"), + flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"), + flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), + flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), + flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"), + flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"), + flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "filter response with specified favicon hash (-mfc 1494302000)"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", - flagSet.IntVarP(&options.Threads, "threads", "t", 50, "Number of threads"), - flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "Maximum requests to send per second"), + flagSet.IntVarP(&options.Threads, "threads", "t", 50, "number of threads to use"), + flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum requests to send per second"), flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), ) createGroup(flagSet, "Misc", "Miscellaneous", - flagSet.BoolVar(&options.Favicon, "favicon", false, "Probes for favicon (\"favicon.ico\" as path) and display phythonic hash"), - flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "Perform TLS(SSL) data grabbing"), - flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "Send HTTP probes on the extracted TLS domains"), - flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "Send HTTP probes on the extracted CSP domains"), - flagSet.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline probe"), - flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe"), - flagSet.BoolVar(&options.VHost, "vhost", false, "VHOST Probe"), - flagSet.VarP(&options.CustomPorts, "ports", "p", "Port to scan (nmap syntax: eg 1,2-10,11)"), - flagSet.StringVar(&options.RequestURIs, "path", "", "File or comma separated paths to request"), - flagSet.StringVar(&options.RequestURIs, "paths", "", "File or comma separated paths to request (deprecated)"), - flagSet.StringVar(&options.Hashes, "hash", "", "Display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"), + flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "probe all the ips associated with same host"), + flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "perform TLS(SSL) data grabbing"), + flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "send http probes on the extracted TLS domains (dns_name)"), + flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "send http probes on the extracted CSP domains"), + flagSet.BoolVar(&options.Pipeline, "pipeline", false, "probe and display server supporting HTTP1.1 pipeline"), + flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "probe and display server supporting HTTP2"), + flagSet.BoolVar(&options.VHost, "vhost", false, "probe and display server supporting VHOST"), + flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg 1,2-10,11)"), + flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"), + flagSet.StringVar(&options.RequestURIs, "paths", "", "path or list of paths to probe (comma-separated, file) (deprecated)"), ) createGroup(flagSet, "output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"), - flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in CSV format"), + flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"), flagSet.BoolVar(&options.JSONOutput, "json", false, "store output in JSONL(ines) format"), flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"), flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"), @@ -314,47 +318,46 @@ func ParseOptions() *Options { ) createGroup(flagSet, "configs", "Configurations", - flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "List of custom resolvers (file or comma separated)"), - flagSet.Var(&options.Allow, "allow", "Allowed list of IP/CIDR's to process (file or comma separated)"), - flagSet.Var(&options.Deny, "deny", "Denied list of IP/CIDR's to process (file or comma separated)"), + flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "list of custom resolvers (file or comma separated)"), + flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), + flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), - flagSet.VarP(&options.CustomHeaders, "header", "H", "Custom Header to send with request"), - flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "HTTP Proxy, eg http://127.0.0.1:8080"), - flagSet.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization"), - flagSet.BoolVar(&options.Resume, "resume", false, "Resume scan using resume.cfg"), - flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "Follow HTTP redirects"), - flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "maxr", 10, "Max number of redirects to follow per host"), - flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "Follow redirects on the same host"), - flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "Get a list of vhosts as input"), - flagSet.StringVar(&options.Methods, "x", "", "Request methods to use, use 'all' to probe all HTTP methods"), - flagSet.StringVar(&options.RequestBody, "body", "", "Post body to include in HTTP request"), - flagSet.BoolVarP(&options.Stream, "stream", "s", false, "Stream mode - start elaborating input targets without sorting"), - flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "Disable dedupe input items (only used with stream mode)"), - flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "Probe all the ips associated with same host"), - flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "Leave default HTTP/HTTPS ports (eg. http://host:80 - https//host:443"), + flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"), + flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"), + flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"), + flagSet.BoolVar(&options.Resume, "resume", false, "resume scan using resume.cfg"), + flagSet.BoolVarP(&options.FollowRedirects, "follow-redirects", "fr", false, "follow http redirects"), + flagSet.IntVarP(&options.MaxRedirects, "max-redirects", "maxr", 10, "max number of redirects to follow per host"), + flagSet.BoolVarP(&options.FollowHostRedirects, "follow-host-redirects", "fhr", false, "follow redirects on the same host"), + flagSet.BoolVar(&options.VHostInput, "vhost-input", false, "get a list of vhosts as input"), + flagSet.StringVar(&options.Methods, "x", "", "request methods to probe, use 'all' to probe all HTTP methods"), + flagSet.StringVar(&options.RequestBody, "body", "", "post body to include in http request"), + flagSet.BoolVarP(&options.Stream, "stream", "s", false, "stream mode - start elaborating input targets without sorting"), + flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "disable dedupe input items (only used with stream mode)"), + flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports (eg. http://host:80 - https//host:443"), ) createGroup(flagSet, "debug", "Debug", - flagSet.BoolVar(&options.Silent, "silent", false, "Silent mode"), - flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "Verbose mode"), - flagSet.BoolVar(&options.Version, "version", false, "Display version"), - flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "Disable color in output"), - flagSet.BoolVar(&options.Debug, "debug", false, "Debug mode"), - flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "Show all sent requests"), - flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "Show all received responses"), - flagSet.BoolVar(&options.ShowStatistics, "stats", false, "Display scan statistic"), + flagSet.BoolVar(&options.Debug, "debug", false, "display request/response content in cli"), + flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "display request content in cli"), + flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "display response content in cli"), + flagSet.BoolVar(&options.Version, "version", false, "display httpx version"), + flagSet.BoolVar(&options.ShowStatistics, "stats", false, "display scan statistic"), + flagSet.BoolVar(&options.Silent, "silent", false, "silent mode"), + flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"), + flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"), ) createGroup(flagSet, "Optimizations", "Optimizations", - flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "Display both probed protocol (HTTPS and HTTP)"), - flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "Probe with input protocol scheme"), - flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "Max error count per host before skipping remaining path/s"), - flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "Skip full port scans for CDNs (only checks for 80,443)"), - flagSet.IntVar(&options.Retries, "retries", 0, "Number of retries"), - flagSet.IntVar(&options.Timeout, "timeout", 5, "Timeout in seconds"), - flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "Max response size to save in bytes"), - flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "Max response size to read in bytes"), + flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "display both probed protocol (HTTPS and HTTP)"), + flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "), + flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"), + flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDNs (only checks for 80,443)"), + flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"), + flagSet.IntVar(&options.Timeout, "timeout", 5, "timeout in seconds"), + flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"), + flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"), ) _ = flagSet.Parse() From acfb1c73abf599eea20affa94e3b430621a2310a Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 1 Mar 2022 14:29:18 +0530 Subject: [PATCH 118/355] removed duplicate (deprecated) flag - paths --- runner/options.go | 1 - 1 file changed, 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 7fac914..741a080 100644 --- a/runner/options.go +++ b/runner/options.go @@ -303,7 +303,6 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.VHost, "vhost", false, "probe and display server supporting VHOST"), flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg 1,2-10,11)"), flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"), - flagSet.StringVar(&options.RequestURIs, "paths", "", "path or list of paths to probe (comma-separated, file) (deprecated)"), ) createGroup(flagSet, "output", "Output", From b14406ed7c3ceb41f8d73b530a30a5a67fce75f5 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 1 Mar 2022 14:37:41 +0530 Subject: [PATCH 119/355] more updates --- README.md | 15 +++++++-------- runner/options.go | 14 +++++++------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index c9d707e..a3ff44a 100644 --- a/README.md +++ b/README.md @@ -111,9 +111,9 @@ MATCHERS: -ml, -match-length string match response with specified content length (-ml 100,102) -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) + -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string match response with specified string (-ms admin) -mr, -match-regex string match response with specified regex (-mr admin) - -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) EXTRACTOR: -er, -extract-regex string display response content for specified regex @@ -121,11 +121,11 @@ EXTRACTOR: FILTERS: -fc, -filter-code string filter response with specified status code (-fc 403,401) -fl, -filter-length string filter response with specified content length (-fl 23,33) - -fs, -filter-string string filter response with specified string (-fs admin) - -fe, -filter-regex string filter response with specified regex (-fe admin) -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) + -fs, -filter-string string filter response with specified string (-fs admin) + -fe, -filter-regex string filter response with specified regex (-fe admin) RATE-LIMIT: -t, -threads int number of threads to use (default 50) @@ -134,15 +134,14 @@ RATE-LIMIT: MISCELLANEOUS: -pa, -probe-all-ips probe all the ips associated with same host - -tls-grab perform TLS(SSL) data grabbing + -p, -ports string[] ports to probe (nmap syntax: eg 1,2-10,11) + -path string path or list of paths to probe (comma-separated, file) -tls-probe send http probes on the extracted TLS domains (dns_name) -csp-probe send http probes on the extracted CSP domains + -tls-grab perform TLS(SSL) data grabbing -pipeline probe and display server supporting HTTP1.1 pipeline -http2 probe and display server supporting HTTP2 -vhost probe and display server supporting VHOST - -p, -ports string[] ports to probe (nmap syntax: eg 1,2-10,11) - -path string path or list of paths to probe (comma-separated, file) - -paths string path or list of paths to probe (comma-separated, file) (deprecated) OUTPUT: -o, -output string file to write output results @@ -171,7 +170,7 @@ CONFIGURATIONS: -body string post body to include in http request -s, -stream stream mode - start elaborating input targets without sorting -sd, -skip-dedupe disable dedupe input items (only used with stream mode) - -ldp, -leave-default-ports leave default http/https ports (eg. http://host:80 - https//host:443 + -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443) DEBUG: -debug display request/response content in cli diff --git a/runner/options.go b/runner/options.go index 741a080..f209775 100644 --- a/runner/options.go +++ b/runner/options.go @@ -268,9 +268,9 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"), flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"), flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"), + flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "match response with specified favicon hash (-mfc 1494302000)"), flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), - flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "match response with specified favicon hash (-mfc 1494302000)"), ) createGroup(flagSet, "extractor", "Extractor", @@ -280,11 +280,11 @@ func ParseOptions() *Options { createGroup(flagSet, "filters", "Filters", flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"), flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"), - flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), - flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"), flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"), flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "filter response with specified favicon hash (-mfc 1494302000)"), + flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), + flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", @@ -295,14 +295,14 @@ func ParseOptions() *Options { createGroup(flagSet, "Misc", "Miscellaneous", flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "probe all the ips associated with same host"), - flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "perform TLS(SSL) data grabbing"), + flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg 1,2-10,11)"), + flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"), flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "send http probes on the extracted TLS domains (dns_name)"), flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "send http probes on the extracted CSP domains"), + flagSet.BoolVar(&options.TLSGrab, "tls-grab", false, "perform TLS(SSL) data grabbing"), flagSet.BoolVar(&options.Pipeline, "pipeline", false, "probe and display server supporting HTTP1.1 pipeline"), flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "probe and display server supporting HTTP2"), flagSet.BoolVar(&options.VHost, "vhost", false, "probe and display server supporting VHOST"), - flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg 1,2-10,11)"), - flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"), ) createGroup(flagSet, "output", "Output", @@ -333,7 +333,7 @@ func ParseOptions() *Options { flagSet.StringVar(&options.RequestBody, "body", "", "post body to include in http request"), flagSet.BoolVarP(&options.Stream, "stream", "s", false, "stream mode - start elaborating input targets without sorting"), flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "disable dedupe input items (only used with stream mode)"), - flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports (eg. http://host:80 - https//host:443"), + flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https//host:443"), ) createGroup(flagSet, "debug", "Debug", From 1c737d7c426300043d6c592d629fa8133000faeb Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 1 Mar 2022 14:43:30 +0530 Subject: [PATCH 120/355] typo update --- README.md | 2 +- runner/options.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a3ff44a..5847f1a 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ OUTPUT: -store-chain include http redirect chain in responses (-sr only) CONFIGURATIONS: - -r, -resolvers string[] list of custom resolvers (file or comma separated) + -r, -resolvers string[] list of custom resolver (file or comma separated) -allow string[] allowed list of IP/CIDR's to process (file or comma separated) -deny string[] denied list of IP/CIDR's to process (file or comma separated) -random-agent Enable Random User-Agent to use (default true) diff --git a/runner/options.go b/runner/options.go index f209775..8b72715 100644 --- a/runner/options.go +++ b/runner/options.go @@ -317,7 +317,7 @@ func ParseOptions() *Options { ) createGroup(flagSet, "configs", "Configurations", - flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "list of custom resolvers (file or comma separated)"), + flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "list of custom resolver (file or comma separated)"), flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), From 05ab529b34d97996c2a1bf48a2cd36198d1ddb04 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 1 Mar 2022 14:53:58 +0530 Subject: [PATCH 121/355] minor breaking change in default json output --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 1608a2f..bb585eb 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.1.6 + /_/ v1.2.0 ` // Version is the current version of httpx -const Version = `v1.1.6` +const Version = `v1.2.0` // showBanner is used to show the banner to the user func showBanner() { From 25d8b18f719c2fbf4c6ea1c52cf7a66c213f298f Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 1 Mar 2022 16:11:05 +0530 Subject: [PATCH 122/355] dev version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index bb585eb..1a1a0be 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.0 + /_/ v1.2.1-dev ` // Version is the current version of httpx -const Version = `v1.2.0` +const Version = `v1.2.0-dev` // showBanner is used to show the banner to the user func showBanner() { From ffd80ae98ffadce42d1e584238bb0af70b419826 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 10:10:07 +0000 Subject: [PATCH 123/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.30 to 0.0.31. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.30...v0.0.31) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f62264d..449ca0f 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.30 + github.com/projectdiscovery/wappalyzergo v0.0.31 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 6908260..6297663 100644 --- a/go.sum +++ b/go.sum @@ -160,8 +160,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.30 h1:DngC7onROGD+ugfo1xV3iRzi8l+dlbO7RxQ5ojZ2zuY= -github.com/projectdiscovery/wappalyzergo v0.0.30/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.31 h1:bWhSCy2l6Nk0mQvIeuF+UnkAp3afMIt1UHx0QNxLbok= +github.com/projectdiscovery/wappalyzergo v0.0.31/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From fed1fbd20a1226d09ce8af87b182b16b6956b1d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 10:12:02 +0000 Subject: [PATCH 124/355] chore(deps): bump golang from 1.17.7-alpine to 1.17.8-alpine Bumps golang from 1.17.7-alpine to 1.17.8-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5dae619..195fed3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.7-alpine AS builder +FROM golang:1.17.8-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 33a025274d28822af62a33dcba546c1fe83dbe86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 10:12:03 +0000 Subject: [PATCH 125/355] chore(deps): bump actions/checkout from 2 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-test.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/dockerhub-push.yml | 2 +- .github/workflows/functional-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release-binary.yml | 2 +- .github/workflows/sonarcloud.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 884a579..1f45b54 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -19,7 +19,7 @@ jobs: go-version: 1.17 - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build run: go build . diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 792785b..7c34372 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 0fd9d87..f253b3d 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Get Github tag id: meta diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 9d48a45..943e484 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -18,7 +18,7 @@ jobs: go-version: 1.17 - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Functional Tests run: | diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 5fbc57f..1531c6b 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.1.0 with: diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 657340d..cf6504a 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: "Check out code" - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 62ad03f..1229930 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -13,7 +13,7 @@ jobs: name: SonarCloud runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis From 4f6473928b8f6acc74b7e33fe85bccdca815a9fa Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 8 Mar 2022 07:41:09 +0530 Subject: [PATCH 126/355] go mod update --- go.mod | 16 ++++++++++++---- go.sum | 46 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 449ca0f..7e7ab13 100644 --- a/go.mod +++ b/go.mod @@ -11,14 +11,14 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.18 - github.com/miekg/dns v1.1.43 // indirect + github.com/miekg/dns v1.1.46 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.3 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e + github.com/projectdiscovery/fastdialer v0.0.15 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21 + github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 @@ -27,7 +27,6 @@ require ( github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 github.com/projectdiscovery/mapcidr v0.0.8 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e - github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a // indirect github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 @@ -61,8 +60,17 @@ require ( github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect + github.com/projectdiscovery/retryabledns v1.0.13 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect + github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect + github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect + github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect + github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect + golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 6297663..f29de59 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,7 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -76,8 +77,9 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/p github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= +github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -86,6 +88,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -100,6 +103,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -113,12 +117,12 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e h1:ORAzxUm2Dodjp2le/OqCD7S5KDqewV0F1REhabb2fD4= -github.com/projectdiscovery/fastdialer v0.0.14-0.20211117222717-6599e7bc586e/go.mod h1:Mex24omi3RxrmhA8Ote7rw+6LWMiaBvbJq8CNp0ksII= +github.com/projectdiscovery/fastdialer v0.0.15 h1:vOLk8jty+1H9osPRzUT6acD0XBSaglyLS3MlIQaYczk= +github.com/projectdiscovery/fastdialer v0.0.15/go.mod h1:Q28lw9oTpiZHq09uFG6YYYLUsUjsOypZ7PXWwQGBB80= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= -github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21 h1:YtKqlzdhAfXbV6R7QrcoaP21+lpb6G++GEDqJbeEmQA= -github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c h1:KDmCXhLLnS/Gc1VDyTxxamRzc8OmHCm1X+f8WQoaTRs= +github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= @@ -146,8 +150,9 @@ github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpG github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a h1:WJQjr9qi/VjWhdNiGyNqcFi0967Gp0W3I769bCpHOJE= github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= +github.com/projectdiscovery/retryabledns v1.0.13 h1:Ogfv0fl3Nszb+Nq2S2qQmE+PJDlStVxyLkmTotMohZY= +github.com/projectdiscovery/retryabledns v1.0.13/go.mod h1:EeqHcAPp0g2GljT4qkxKSAE47Dj0ZrJQ46R9ct3Muhk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= @@ -169,6 +174,7 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -177,6 +183,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -185,9 +193,19 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= +github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= +github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 h1:DyAZOw3JsVd6LJHqhl4MpKQdYQEmat0C6pPPwom39Ow= +github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= +github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= +github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b h1:iYQzlljG1dOXBtsJGyzFC/wBK5qUCWs1eLCr/UcJYPA= +github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= @@ -196,25 +214,33 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -222,12 +248,14 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -241,8 +269,11 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -254,9 +285,12 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 742f68e483b842e1b9ef14cf40662c4fbc3e65c2 Mon Sep 17 00:00:00 2001 From: Collins Huff Date: Fri, 11 Mar 2022 10:12:58 -0500 Subject: [PATCH 127/355] add line to include request body in options struct --- runner/runner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/runner/runner.go b/runner/runner.go index c7bcf0a..bc6dd4a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -150,6 +150,7 @@ func New(options *Options) (*Runner, error) { } scanopts.RequestBody = rrBody options.rawRequest = string(rawRequest) + options.RequestBody = rrBody } // disable automatic host header for rawhttp if manually specified From e0e6f1806b38d4df0859c923c04b5c13b2cec33a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Mar 2022 10:12:19 +0000 Subject: [PATCH 128/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.31 to 0.0.32. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.31...v0.0.32) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7e7ab13..2a5ff5d 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.31 + github.com/projectdiscovery/wappalyzergo v0.0.32 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index f29de59..da956bc 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.31 h1:bWhSCy2l6Nk0mQvIeuF+UnkAp3afMIt1UHx0QNxLbok= -github.com/projectdiscovery/wappalyzergo v0.0.31/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.32 h1:fbbQw68gFqT8KqVcYAIsoH3E4XQjz3uZyyeGLbbYBIk= +github.com/projectdiscovery/wappalyzergo v0.0.32/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From faab923bd45538c7ee24154be9b654a69ba6dff9 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 15 Mar 2022 15:28:45 -0500 Subject: [PATCH 129/355] display first title --- common/httpx/title.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/httpx/title.go b/common/httpx/title.go index 125f82f..88a2b34 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -90,7 +90,7 @@ func getTitleWithDom(r *Response) (*html.Node, error) { title = node return } - for child := node.FirstChild; child != nil; child = child.NextSibling { + for child := node.FirstChild; child != nil && title == nil; child = child.NextSibling { crawler(child) } } From 23cb073cd629e9e756d9dfa03a8a047ba7e27460 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 21 Mar 2022 15:31:56 +0530 Subject: [PATCH 130/355] lint workflow update --- .github/workflows/lint-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 1531c6b..ae50778 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -9,6 +9,10 @@ jobs: name: Lint Test runs-on: ubuntu-latest steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint From d750d70f58d8a8c7abace16cc5e9d36e2026ce5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 10:05:50 +0000 Subject: [PATCH 131/355] chore(deps): bump alpine from 3.15.0 to 3.15.1 Bumps alpine from 3.15.0 to 3.15.1. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 195fed3..7221a03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.8-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.15.0 +FROM alpine:3.15.1 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From f4688f452d1d87ae415f0b49618ac3c55c042471 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 10:05:51 +0000 Subject: [PATCH 132/355] chore(deps): bump golang from 1.17.8-alpine to 1.18.0-alpine Bumps golang from 1.17.8-alpine to 1.18.0-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 195fed3..cb951ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.8-alpine AS builder +FROM golang:1.18.0-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 90caade6e56a1f88ba0fdd12ec634293c65b631f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 10:38:28 +0000 Subject: [PATCH 133/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.32 to 0.0.33. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.32...v0.0.33) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2a5ff5d..c690ae4 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.32 + github.com/projectdiscovery/wappalyzergo v0.0.33 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.3.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index da956bc..209ce80 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xb github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.32 h1:fbbQw68gFqT8KqVcYAIsoH3E4XQjz3uZyyeGLbbYBIk= -github.com/projectdiscovery/wappalyzergo v0.0.32/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.33 h1:vlKAil++KSdJ0vkX7/3Xiom0R6L1aeoYDbFITVcYCZs= +github.com/projectdiscovery/wappalyzergo v0.0.33/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From ffb2079dfb2a8740000d938575d49facd6ae395a Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 22 Mar 2022 16:47:55 +0100 Subject: [PATCH 134/355] Adding CSP parsing form HTML meta --- common/httpx/csp.go | 45 ++++++++++++++++++++++++++++++------------- common/httpx/httpx.go | 2 +- go.mod | 4 +++- go.sum | 6 ++++++ 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/common/httpx/csp.go b/common/httpx/csp.go index 7a68ac9..e1c4831 100644 --- a/common/httpx/csp.go +++ b/common/httpx/csp.go @@ -1,10 +1,12 @@ package httpx import ( - "net/http" + "bytes" "strings" + "github.com/PuerkitoBio/goquery" "github.com/projectdiscovery/httpx/common/slice" + "github.com/projectdiscovery/stringsutil" ) // CSPHeaders is an incomplete list of most common CSP headers @@ -21,22 +23,28 @@ type CSPData struct { } // CSPGrab fills the CSPData -func (h *HTTPX) CSPGrab(r *http.Response) *CSPData { +func (h *HTTPX) CSPGrab(r *Response) *CSPData { domains := make(map[string]struct{}) + // extract from headers for _, cspHeader := range CSPHeaders { - cspRaw := r.Header.Get(cspHeader) - if cspRaw != "" { - rules := strings.Split(cspRaw, ";") - for _, rule := range rules { - // rule is like aa bb domain1 domain2 domain3 - tokens := strings.Split(rule, " ") - // we extracts only potential domains - for _, t := range tokens { - if isPotentialDomain(t) { - domains[t] = struct{}{} + if cspValues, ok := r.Headers[cspHeader]; ok { + for _, cspValue := range cspValues { + parsePotentialDomains(domains, cspValue) + } + } + } + + // extract from body + if len(r.Data) > 0 { + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(r.Data)) + if err == nil { + doc.Find("meta").Each(func(i int, s *goquery.Selection) { + if _, ok := s.Attr("http-equiv"); ok { + if content, ok := s.Attr("content"); ok { + parsePotentialDomains(domains, content) } } - } + }) } } @@ -46,6 +54,17 @@ func (h *HTTPX) CSPGrab(r *http.Response) *CSPData { return nil } +func parsePotentialDomains(domains map[string]struct{}, data string) { + // rule is like aa bb domain1 domain2 domain3 + tokens := stringsutil.SplitAny(data, " ", ";", ",") + // we extracts only potential domains + for _, t := range tokens { + if isPotentialDomain(t) { + domains[t] = struct{}{} + } + } +} + func isPotentialDomain(s string) bool { return strings.Contains(s, ".") || strings.HasPrefix(s, "http") } diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index a0cc15d..d2b159a 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -230,7 +230,7 @@ get_response: resp.TLSData = h.TLSGrab(httpresp) } - resp.CSPData = h.CSPGrab(httpresp) + resp.CSPData = h.CSPGrab(&resp) // build the redirect flow by reverse cycling the response<-request chain if !h.Options.Unsafe { diff --git a/go.mod b/go.mod index c690ae4..f45e6e8 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea - github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 + github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 github.com/projectdiscovery/wappalyzergo v0.0.33 github.com/remeh/sizedwaitgroup v1.0.0 @@ -48,7 +48,9 @@ require github.com/spaolacci/murmur3 v1.1.0 require github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect diff --git a/go.sum b/go.sum index 209ce80..4d760cb 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,15 @@ github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -163,6 +167,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4fvUw6b4sS4GoWbHr60mJo3dI//4mGt3BuLx8Sz9aNw= +github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= github.com/projectdiscovery/wappalyzergo v0.0.33 h1:vlKAil++KSdJ0vkX7/3Xiom0R6L1aeoYDbFITVcYCZs= From 3cac25132fedcf64f57ad026bea7622e85846bc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Mar 2022 10:35:40 +0000 Subject: [PATCH 135/355] chore(deps): bump github.com/rs/xid from 1.3.0 to 1.4.0 Bumps [github.com/rs/xid](https://github.com/rs/xid) from 1.3.0 to 1.4.0. - [Release notes](https://github.com/rs/xid/releases) - [Commits](https://github.com/rs/xid/compare/v1.3.0...v1.4.0) --- updated-dependencies: - dependency-name: github.com/rs/xid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c690ae4..ec61960 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 github.com/projectdiscovery/wappalyzergo v0.0.33 github.com/remeh/sizedwaitgroup v1.0.0 - github.com/rs/xid v1.3.0 + github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect diff --git a/go.sum b/go.sum index 209ce80..6e306ac 100644 --- a/go.sum +++ b/go.sum @@ -172,8 +172,8 @@ github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNC github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= From e4fcb54833695ab4c5afbf2523adb29fa781507b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 10:05:14 +0000 Subject: [PATCH 136/355] chore(deps): bump alpine from 3.15.1 to 3.15.2 Bumps alpine from 3.15.1 to 3.15.2. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef75f2e..e41394b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.18.0-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.15.1 +FROM alpine:3.15.2 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From 9815c5754e367b1efee07d3c83b19787daa0a7af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 10:14:05 +0000 Subject: [PATCH 137/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.33 to 0.0.34. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.33...v0.0.34) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 +++++--- go.sum | 5 ++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0489579..e039710 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.33 + github.com/projectdiscovery/wappalyzergo v0.0.34 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect @@ -45,10 +45,12 @@ require ( require github.com/spaolacci/murmur3 v1.1.0 -require github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 +require ( + github.com/PuerkitoBio/goquery v1.8.0 + github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 +) require ( - github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 4a52725..9b40de1 100644 --- a/go.sum +++ b/go.sum @@ -165,14 +165,13 @@ github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4fvUw6b4sS4GoWbHr60mJo3dI//4mGt3BuLx8Sz9aNw= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.33 h1:vlKAil++KSdJ0vkX7/3Xiom0R6L1aeoYDbFITVcYCZs= -github.com/projectdiscovery/wappalyzergo v0.0.33/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.34 h1:xYLXTPkU6gcuHHEuxClReMLgbCkAbOjl1accgji3Gtc= +github.com/projectdiscovery/wappalyzergo v0.0.34/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 987131a20be7ffcc7f917af52e862191aa945137 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 29 Mar 2022 19:34:55 +0200 Subject: [PATCH 138/355] Adding custom cookies supports in redirects --- common/httpx/httpx.go | 25 ++++++++++++++++++++++--- common/httpx/option.go | 17 +++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index d2b159a..23c0461 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -55,6 +55,8 @@ func New(options *Options) (*HTTPX, error) { httpx.Options = options + httpx.Options.parseCustomCookies() + var retryablehttpOptions = retryablehttp.DefaultOptionsSpraying retryablehttpOptions.Timeout = httpx.Options.Timeout retryablehttpOptions.RetryMax = httpx.Options.RetryMax @@ -67,6 +69,8 @@ func New(options *Options) (*HTTPX, error) { if httpx.Options.FollowRedirects { // Follow redirects up to a maximum number redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { + // add custom cookies if necessary + httpx.setCustomCookies(redirectedRequest) if len(previousRequests) >= options.MaxRedirects { // https://github.com/golang/go/issues/10069 return http.ErrUseLastResponse @@ -78,6 +82,9 @@ func New(options *Options) (*HTTPX, error) { if httpx.Options.FollowHostRedirects { // Only follow redirects on the same host up to a maximum number redirectFunc = func(redirectedRequest *http.Request, previousRequests []*http.Request) error { + // add custom cookies if necessary + httpx.setCustomCookies(redirectedRequest) + // Check if we get a redirect to a different host var newHost = redirectedRequest.URL.Host var oldHost = previousRequests[0].Host @@ -322,13 +329,25 @@ func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL str // SetCustomHeaders on the provided request func (h *HTTPX) SetCustomHeaders(r *retryablehttp.Request, headers map[string]string) { for name, value := range headers { - r.Header.Set(name, value) - // host header is particular - if strings.EqualFold(name, "host") { + switch strings.ToLower(name) { + case "host": r.Host = value + case "cookie": + // cookies are set in the default branch, and reset during the follow redirect flow + fallthrough + default: + r.Header.Set(name, value) } } if h.Options.RandomAgent { r.Header.Set("User-Agent", uarand.GetRandom()) //nolint } } + +func (httpx *HTTPX) setCustomCookies(req *http.Request) { + if httpx.Options.hasCustomCookies() { + for _, cookie := range httpx.Options.customCookies { + req.AddCookie(cookie) + } + } +} diff --git a/common/httpx/option.go b/common/httpx/option.go index 36d834a..059665a 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -1,6 +1,8 @@ package httpx import ( + "net/http" + "strings" "time" ) @@ -37,6 +39,7 @@ type Options struct { MaxResponseBodySizeToRead int64 UnsafeURI string Resolvers []string + customCookies []*http.Cookie } // DefaultOptions contains the default options @@ -58,3 +61,17 @@ var DefaultOptions = Options{ VHostSimilarityRatio: 85, DefaultUserAgent: "httpx - Open-source project (github.com/projectdiscovery/httpx)", } + +func (options *Options) parseCustomCookies() { + // parse and fill the custom field + for k, v := range options.CustomHeaders { + if strings.EqualFold(k, "cookie") { + req := http.Request{Header: http.Header{"Cookie": []string{v}}} + options.customCookies = req.Cookies() + } + } +} + +func (options *Options) hasCustomCookies() bool { + return len(options.customCookies) > 0 +} From e7d4f46406924b20fb5c5349a4d4974d402a7f9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Apr 2022 10:06:29 +0000 Subject: [PATCH 139/355] chore(deps): bump alpine from 3.15.2 to 3.15.3 Bumps alpine from 3.15.2 to 3.15.3. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e41394b..30483e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.18.0-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.15.2 +FROM alpine:3.15.3 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From ea4bff53e82e0a98cddae10e5badba8d6a0695d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Apr 2022 10:12:33 +0000 Subject: [PATCH 140/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.34 to 0.0.35. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.34...v0.0.35) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e039710..a452fb1 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.34 + github.com/projectdiscovery/wappalyzergo v0.0.35 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 9b40de1..e3757ca 100644 --- a/go.sum +++ b/go.sum @@ -170,8 +170,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.34 h1:xYLXTPkU6gcuHHEuxClReMLgbCkAbOjl1accgji3Gtc= -github.com/projectdiscovery/wappalyzergo v0.0.34/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.35 h1:UDjCmOygrY0Q25ZH4jz5pEw67wPxO6ilHYJQoxgdfu4= +github.com/projectdiscovery/wappalyzergo v0.0.35/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From df5f69fd18eeb7615447dff3357c9a0396fb7d0c Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 7 Apr 2022 02:12:09 -0500 Subject: [PATCH 141/355] Issue 495 ASN support (#562) * asn support * minor change * misc * lookup changes * nil pointer check * converted errors into warning --- go.mod | 1 + go.sum | 6 ++++++ runner/options.go | 2 ++ runner/runner.go | 38 +++++++++++++++++++++++++++++++++++++- 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a452fb1..97619cd 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( ) require ( + github.com/ammario/ipisp/v2 v2.0.0 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index e3757ca..dfbac3f 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,10 @@ github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejn github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= +github.com/ammario/ipisp v1.0.0 h1:U4xdVMBFWm0/4sHrQ3hVMC+ygg/Ynm4/vdFdkVAex1o= +github.com/ammario/ipisp v1.0.0/go.mod h1:HM60VFpmEWyU+FisnTTHCeswaU3RW0dCVHihgIGUEGM= +github.com/ammario/ipisp/v2 v2.0.0 h1:/aRMp5srZViiBfOUGzl/Esqae4s0MDDzm9buhGcZ0XU= +github.com/ammario/ipisp/v2 v2.0.0/go.mod h1:bQ6KAL5LnYYEj6olUn+Bzv/im/4Esa5oGkbv9b+uOjo= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= @@ -109,6 +113,7 @@ github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -189,6 +194,7 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/runner/options.go b/runner/options.go index 8b72715..08c14ef 100644 --- a/runner/options.go +++ b/runner/options.go @@ -228,6 +228,7 @@ type Options struct { OutputFilterWordsCount string filterWordsCount []int Hashes string + Asn bool } // ParseOptions parses the command line options for application @@ -314,6 +315,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"), flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"), flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"), + flagSet.BoolVar(&options.Asn, "asn", false, "displays asn information"), ) createGroup(flagSet, "configs", "Configurations", diff --git a/runner/runner.go b/runner/runner.go index bc6dd4a..c96ba22 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -22,6 +22,7 @@ import ( "strings" "time" + "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -957,7 +958,6 @@ retry: builder.WriteRune(']') } - if err != nil { errString := "" errString = err.Error() @@ -1129,6 +1129,29 @@ retry: } } ip := hp.Dialer.GetDialedIP(URL.Host) + var asnResponse interface{ String() string } + if r.options.Asn { + lookupResult, err := ipisp.LookupIP(context.Background(), net.ParseIP(ip)) + if err != nil { + gologger.Warning().Msg(err.Error()) + } + if lookupResult != nil { + lookupResult.ISPName = stringsutil.TrimSuffixAny(strings.ReplaceAll(lookupResult.ISPName, lookupResult.Country, ""), ", ", " ") + asnResponse = AsnResponse{ + AsNumber: lookupResult.ASN.String(), + AsName: lookupResult.ISPName, + AsCountry: lookupResult.Country, + AsRange: lookupResult.Range.String(), + } + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(asnResponse.String()).String()) + } else { + builder.WriteString(asnResponse.String()) + } + builder.WriteRune(']') + } + } // hp.Dialer.GetDialedIP would return only the last dialed one if customIP != "" { ip = customIP @@ -1376,6 +1399,7 @@ retry: Hashes: hashesMap, Lines: resp.Lines, Words: resp.Words, + ASN: asnResponse, } } @@ -1387,6 +1411,17 @@ func (r *Runner) SaveResumeConfig() error { return goconfig.Save(resumeCfg, DefaultResumeFile) } +type AsnResponse struct { + AsNumber string `json:"as-number" csv:"as-number"` + AsName string `json:"as-name" csv:"as-name"` + AsCountry string `json:"as-country" csv:"as-country"` + AsRange string `json:"as-range" csv:"as-range"` +} + +func (o AsnResponse) String() string { + return fmt.Sprintf("%v, %v, %v, %v", o.AsNumber, o.AsName, o.AsCountry, o.AsRange) +} + // Result of a scan type Result struct { Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` @@ -1428,6 +1463,7 @@ type Result struct { Failed bool `json:"failed" csv:"failed"` FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` + ASN interface{} `json:"asn,omitempty" csv:"asn"` Lines int `json:"lines" csv:"lines"` Words int `json:"words" csv:"words"` } From 48d9552480a9d3c294d3fe0053222b91ba8f18a0 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Thu, 7 Apr 2022 16:18:11 +0530 Subject: [PATCH 142/355] Added snapcraft config file --- snapcraft.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 snapcraft.yaml diff --git a/snapcraft.yaml b/snapcraft.yaml new file mode 100644 index 0000000..84e100d --- /dev/null +++ b/snapcraft.yaml @@ -0,0 +1,34 @@ +name: httpx +summary: httpx is a fast and multi-purpose HTTP toolkit +description: | + httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library, it is designed to maintain the result reliability with increased threads. +version: 'v1.2.1' +icon: images/snapcraft_icon.png +license: MIT +base: core20 +grade: stable +confinement: strict +architectures: + - build-on: [s390x, ppc64el, arm64, armhf and amd64] + run-on: [s390x, ppc64el, arm64, armhf and amd64] + +apps: + httpx: + command: bin/httpx + plugs: + - home + - network + - removable-media + - dot-httpx + +plugs: + dot-httpx: + interface: personal-files + write: + - $HOME/.config/httpx + +parts: + httpx: + source: https://github.com/projectdiscovery/httpx.git + plugin: go + go-channel: latest/stable From 87b9ea95dae73cb7719075d461ab72ac9d189074 Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 7 Apr 2022 17:08:33 +0530 Subject: [PATCH 143/355] invalid image update --- snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index 84e100d..fd128a4 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -3,7 +3,7 @@ summary: httpx is a fast and multi-purpose HTTP toolkit description: | httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library, it is designed to maintain the result reliability with increased threads. version: 'v1.2.1' -icon: images/snapcraft_icon.png +icon: static/httpx-logo.png license: MIT base: core20 grade: stable From ccf0c9ddc5b9209ea565c6853b78bb253dcf5c47 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Fri, 8 Apr 2022 08:20:03 -0500 Subject: [PATCH 144/355] Issue 549 jarm algorithm (#567) * jarm hash * lint error fix * remove default jarm value if jarm not calculated * jarm cli boolean flag * removed unused hash type * readme update * example update Co-authored-by: Sandeep Singh --- README.md | 29 ++++++++++++ common/hashes/hashes.go | 1 + common/hashes/jarmhash.go | 93 +++++++++++++++++++++++++++++++++++++ common/regexhelper/regex.go | 7 +++ go.mod | 1 + go.sum | 10 ++++ runner/options.go | 5 +- runner/runner.go | 14 +++++- 8 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 common/hashes/jarmhash.go create mode 100644 common/regexhelper/regex.go diff --git a/README.md b/README.md index 5847f1a..6fed2e7 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi | Favicon Hash | false | Probe Status | false | | Body Hash | true | Header Hash | true | | Redirect chain | false | URL Scheme | true | +| JARM Hash | false | | | # Installation Instructions @@ -93,6 +94,7 @@ PROBES: -location display response redirect location -favicon display mmh3 hash for '/favicon.ico' file -hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512) + -jarm display jarm fingerprint hash -rt, -response-time display response time -lc, -line-count display response body line count -wc, -word-count display response body word count @@ -334,6 +336,33 @@ https://mta-sts.hackerone.com/favicon.ico [-1700323260] https://www.hackerone.com/favicon.ico [778073381] ``` +### [JARM Fingerprint](https://github.com/salesforce/jarm) + + +```console +subfinder -d hackerone.com -silent | httpx -jarm + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ v1.2.1 + + projectdiscovery.io + +Use with caution. You are responsible for your actions. +Developers assume no liability and are not responsible for any misuse or damage. +https://www.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007] +https://mta-sts.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] +https://mta-sts.managed.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] +https://docs.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] +https://support.hackerone.com [29d3dd00029d29d00029d3dd29d29d5a74e95248e58a6162e37847a24849f7] +https://api.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007] +https://mta-sts.forwarding.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af] +https://resources.hackerone.com [2ad2ad0002ad2ad0002ad2ad2ad2ad043bfbd87c13813505a1b60adf4f6ff5] +``` + + ### Path Probe diff --git a/common/hashes/hashes.go b/common/hashes/hashes.go index 884968d..5a2a160 100644 --- a/common/hashes/hashes.go +++ b/common/hashes/hashes.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "github.com/mfonda/simhash" "github.com/spaolacci/murmur3" ) diff --git a/common/hashes/jarmhash.go b/common/hashes/jarmhash.go new file mode 100644 index 0000000..e1d48d1 --- /dev/null +++ b/common/hashes/jarmhash.go @@ -0,0 +1,93 @@ +package hashes + +import ( + "fmt" + "net" + "net/url" + "strconv" + "strings" + "time" + + "github.com/RumbleDiscovery/jarm-go" + "github.com/projectdiscovery/httpx/common/regexhelper" + "golang.org/x/net/proxy" +) + +const defaultPort int = 443 + +var DefualtBackoff = func(r, m int) time.Duration { + return time.Second +} + +type target struct { + Host string + Port int + Retries int + Backoff func(r, m int) time.Duration +} + +// fingerprint probes a single host/port +func fingerprint(t target, duration int) string { + timeout := time.Duration(duration) * time.Second + results := []string{} + for _, probe := range jarm.GetProbes(t.Host, t.Port) { + dialer := proxy.FromEnvironmentUsing(&net.Dialer{Timeout: timeout}) + addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port)) + c := net.Conn(nil) + n := 0 + for c == nil && n <= t.Retries { + // Ignoring error since error message was already being dropped. + // Also, if theres an error, c == nil. + if c, _ = dialer.Dial("tcp", addr); c != nil || t.Retries == 0 { + break + } + bo := t.Backoff + if bo == nil { + bo = DefualtBackoff + } + time.Sleep(bo(n, t.Retries)) + n++ + } + if c == nil { + return "" + } + data := jarm.BuildProbe(probe) + _ = c.SetWriteDeadline(time.Now().Add(timeout)) + _, err := c.Write(data) + if err != nil { + results = append(results, "") + c.Close() + continue + } + _ = c.SetReadDeadline(time.Now().Add(timeout)) + buff := make([]byte, 1484) + _, _ = c.Read(buff) + c.Close() + ans, err := jarm.ParseServerHello(buff, probe) + if err != nil { + results = append(results, "") + continue + } + results = append(results, ans) + } + return jarm.RawHashToFuzzyHash(strings.Join(results, ",")) +} +func Jarm(host string, duration int) string { + t := target{} + if u, err := url.Parse(host); err == nil { + if u.Scheme == "http" { + return "" + } + t.Host = u.Hostname() + port, _ := strconv.Atoi(u.Port()) + t.Port = port + } + if t.Port == 0 { + t.Port = defaultPort + } + hash := fingerprint(t, duration) + if regexhelper.JarmHashRegex.MatchString(hash) { + return "" + } + return hash +} diff --git a/common/regexhelper/regex.go b/common/regexhelper/regex.go new file mode 100644 index 0000000..7bdd81a --- /dev/null +++ b/common/regexhelper/regex.go @@ -0,0 +1,7 @@ +package regexhelper + +import "regexp" + +var ( + JarmHashRegex = regexp.MustCompile("(?m)0{62}") +) diff --git a/go.mod b/go.mod index 97619cd..f98b287 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( ) require ( + github.com/RumbleDiscovery/jarm-go v0.0.6 // indirect github.com/ammario/ipisp/v2 v2.0.0 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect diff --git a/go.sum b/go.sum index dfbac3f..986f755 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,9 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= +github.com/RumbleDiscovery/jarm-go v0.0.6/go.mod h1:dXV7z5vBXQI0cNaHXwzGtq2PJ2LgM3XgcFiX32FU3bg= +github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= @@ -32,6 +35,7 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -85,6 +89,7 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/p github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= @@ -185,6 +190,7 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -228,6 +234,7 @@ go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6m golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= @@ -243,6 +250,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -256,6 +264,7 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcx golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -270,6 +279,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/runner/options.go b/runner/options.go index 08c14ef..602faf0 100644 --- a/runner/options.go +++ b/runner/options.go @@ -1,12 +1,11 @@ package runner import ( - "github.com/projectdiscovery/httpx/common/slice" "math" "os" "regexp" "strings" - + "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/goflags" @@ -228,6 +227,7 @@ type Options struct { OutputFilterWordsCount string filterWordsCount []int Hashes string + Jarm bool Asn bool } @@ -250,6 +250,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Location, "location", false, "display response redirect location"), flagSet.BoolVar(&options.Favicon, "favicon", false, "display mmh3 hash for '/favicon.ico' file"), flagSet.StringVar(&options.Hashes, "hash", "", "display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)"), + flagSet.BoolVar(&options.Jarm, "jarm", false, "display jarm fingerprint hash"), flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "display response time"), flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "display response body line count"), flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "display response body word count"), diff --git a/runner/runner.go b/runner/runner.go index c96ba22..73c0270 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1291,7 +1291,17 @@ retry: } builder.WriteRune(']') } - + jarmhash := "" + if r.options.Jarm { + jarmhash = hashes.Jarm(fullURL,r.options.Timeout) + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(jarmhash).String()) + } else { + builder.WriteString(fmt.Sprint(jarmhash)) + } + builder.WriteRune(']') + } if scanopts.OutputWordsCount { builder.WriteString(" [") if !scanopts.OutputWithNoColor { @@ -1397,6 +1407,7 @@ retry: FinalURL: finalURL, FavIconMMH3: faviconMMH3, Hashes: hashesMap, + Jarm: jarmhash, Lines: resp.Lines, Words: resp.Words, ASN: asnResponse, @@ -1466,6 +1477,7 @@ type Result struct { ASN interface{} `json:"asn,omitempty" csv:"asn"` Lines int `json:"lines" csv:"lines"` Words int `json:"words" csv:"words"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` } // JSON the result From 5e9fb63beac9a6b12c25108ab1e3d026df80ded2 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 8 Apr 2022 18:58:37 +0530 Subject: [PATCH 145/355] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 1a1a0be..47b86df 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.1-dev + /_/ v1.2.1 ` // Version is the current version of httpx -const Version = `v1.2.0-dev` +const Version = `v1.2.1` // showBanner is used to show the banner to the user func showBanner() { From aa9a2c494407f9132cddc60ad18089be25097873 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 8 Apr 2022 19:22:36 +0530 Subject: [PATCH 146/355] readme update --- README.md | 4 ++-- runner/options.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6fed2e7..6e45181 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,7 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi | Favicon Hash | false | Probe Status | false | | Body Hash | true | Header Hash | true | | Redirect chain | false | URL Scheme | true | -| JARM Hash | false | | | - +| JARM Hash | false | ASN | false | # Installation Instructions @@ -105,6 +104,7 @@ PROBES: -websocket display server using websocket -ip display host ip -cname display host cname + -asn display host asn information -cdn display cdn in use -probe display probe status diff --git a/runner/options.go b/runner/options.go index 602faf0..509b6bc 100644 --- a/runner/options.go +++ b/runner/options.go @@ -261,6 +261,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"), flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"), flagSet.BoolVar(&options.OutputCName, "cname", false, "display host cname"), + flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"), flagSet.BoolVar(&options.OutputCDN, "cdn", false, "display cdn in use"), flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), ) @@ -316,7 +317,6 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"), flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"), flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"), - flagSet.BoolVar(&options.Asn, "asn", false, "displays asn information"), ) createGroup(flagSet, "configs", "Configurations", From 3bb618440ea4a33c26d77da797e0db4c93b36bab Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 8 Apr 2022 19:28:22 +0530 Subject: [PATCH 147/355] Added ASN readme example --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index 6e45181..40f944c 100644 --- a/README.md +++ b/README.md @@ -362,6 +362,33 @@ https://mta-sts.forwarding.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a https://resources.hackerone.com [2ad2ad0002ad2ad0002ad2ad2ad2ad043bfbd87c13813505a1b60adf4f6ff5] ``` +### ASN Fingerprint + + +```console +subfinder -d hackerone.com -silent | httpx -asn + __ __ __ _ __ + / /_ / /_/ /_____ | |/ / + / __ \/ __/ __/ __ \| / + / / / / /_/ /_/ /_/ / | +/_/ /_/\__/\__/ .___/_/|_| + /_/ v1.2.1 + + projectdiscovery.io + +Use with caution. You are responsible for your actions. +Developers assume no liability and are not responsible for any misuse or damage. +https://mta-sts.managed.hackerone.com [AS54113, FASTLY, US, 185.199.108.0/24] +https://gslink.hackerone.com [AS16509, AMAZON-02, US, 13.33.168.0/22] +https://www.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20] +https://mta-sts.forwarding.hackerone.com [AS54113, FASTLY, US, 185.199.108.0/24] +https://resources.hackerone.com [AS16509, AMAZON-02, US, 3.98.0.0/15] +https://support.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.48.0/20] +https://mta-sts.hackerone.com [AS54113, FASTLY, US, 185.199.111.0/24] +https://docs.hackerone.com [AS54113, FASTLY, US, 185.199.109.0/24] +https://api.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20] +``` + ### Path Probe From c201cbb6d9e34d6fc46292946c0f42e5795ff330 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Fri, 8 Apr 2022 20:45:49 +0530 Subject: [PATCH 148/355] snapcraft config update --- snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index fd128a4..a531459 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -5,7 +5,7 @@ description: | version: 'v1.2.1' icon: static/httpx-logo.png license: MIT -base: core20 +base: core18 grade: stable confinement: strict architectures: From b117023d63952e379fbd74d6a2578a467c226df8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 10:08:25 +0000 Subject: [PATCH 149/355] chore(deps): bump actions/setup-go from 2 to 3 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 2 to 3. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-test.yml | 2 +- .github/workflows/functional-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release-binary.yml | 2 +- .github/workflows/sonarcloud.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1f45b54..b17ce23 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -14,7 +14,7 @@ jobs: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.17 diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 943e484..bea5fc4 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -13,7 +13,7 @@ jobs: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.17 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index ae50778..0ac776b 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.17 - name: Checkout code diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index cf6504a..1ab659c 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -15,7 +15,7 @@ jobs: fetch-depth: 0 - name: "Set up Go" - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.17 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 1229930..b64d6ef 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -18,7 +18,7 @@ jobs: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: "Set up Go" - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.17 From 7d495a0157a2ecae5af3c2d7db9aac3bacdf2470 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 10:08:28 +0000 Subject: [PATCH 150/355] chore(deps): bump alpine from 3.15.3 to 3.15.4 Bumps alpine from 3.15.3 to 3.15.4. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 30483e4..e2880bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.18.0-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.15.3 +FROM alpine:3.15.4 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From aa0a851a77b748ea6002de483c2ce84187f8560a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Apr 2022 10:13:59 +0000 Subject: [PATCH 151/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.35 to 0.0.36. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.35...v0.0.36) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index f98b287..561e69d 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.35 + github.com/projectdiscovery/wappalyzergo v0.0.36 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect @@ -47,12 +47,12 @@ require github.com/spaolacci/murmur3 v1.1.0 require ( github.com/PuerkitoBio/goquery v1.8.0 + github.com/RumbleDiscovery/jarm-go v0.0.6 + github.com/ammario/ipisp/v2 v2.0.0 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 ) require ( - github.com/RumbleDiscovery/jarm-go v0.0.6 // indirect - github.com/ammario/ipisp/v2 v2.0.0 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 986f755..0e9450a 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,6 @@ github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mo github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/ammario/ipisp v1.0.0 h1:U4xdVMBFWm0/4sHrQ3hVMC+ygg/Ynm4/vdFdkVAex1o= -github.com/ammario/ipisp v1.0.0/go.mod h1:HM60VFpmEWyU+FisnTTHCeswaU3RW0dCVHihgIGUEGM= github.com/ammario/ipisp/v2 v2.0.0 h1:/aRMp5srZViiBfOUGzl/Esqae4s0MDDzm9buhGcZ0XU= github.com/ammario/ipisp/v2 v2.0.0/go.mod h1:bQ6KAL5LnYYEj6olUn+Bzv/im/4Esa5oGkbv9b+uOjo= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= @@ -118,7 +116,6 @@ github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -180,8 +177,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.35 h1:UDjCmOygrY0Q25ZH4jz5pEw67wPxO6ilHYJQoxgdfu4= -github.com/projectdiscovery/wappalyzergo v0.0.35/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.36 h1:8wz4ZvOP4FHMOMn7r4BSDPmQGFAbJk6gNx9w3elsv3A= +github.com/projectdiscovery/wappalyzergo v0.0.36/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -200,7 +197,6 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= From d3c485c3ce713424e13cd0dd980c0f5d6f6bd56d Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Wed, 13 Apr 2022 13:29:52 -0500 Subject: [PATCH 152/355] match/filter cdn --- README.md | 2 ++ go.mod | 2 +- go.sum | 2 ++ runner/options.go | 11 +++++++++++ runner/runner.go | 8 +++++++- 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40f944c..2c20964 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ MATCHERS: -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string match response with specified string (-ms admin) -mr, -match-regex string match response with specified regex (-mr admin) + -mcdn, -match-cdn string[] match host with specified cdn provider (-mcdn azure,google) EXTRACTOR: -er, -extract-regex string display response content for specified regex @@ -128,6 +129,7 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) -fs, -filter-string string filter response with specified string (-fs admin) -fe, -filter-regex string filter response with specified regex (-fe admin) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (-fcdn oracle,cloudflare) RATE-LIMIT: -t, -threads int number of threads to use (default 50) diff --git a/go.mod b/go.mod index 561e69d..737a37a 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.18 github.com/miekg/dns v1.1.46 // indirect github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/cdncheck v0.0.3 + github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.15 diff --git a/go.sum b/go.sum index 0e9450a..8601125 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,8 @@ github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwa github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g= github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= +github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= +github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= diff --git a/runner/options.go b/runner/options.go index 509b6bc..b8634c1 100644 --- a/runner/options.go +++ b/runner/options.go @@ -1,11 +1,13 @@ package runner import ( + "fmt" "math" "os" "regexp" "strings" "github.com/projectdiscovery/httpx/common/slice" + "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/goflags" @@ -27,6 +29,8 @@ const ( DefaultOutputDirectory = "output" ) +var defaultProviders = strings.Join(cdncheck.GetDefaultProviders(), ", ") + type scanOptions struct { Methods []string StoreResponseDirectory string @@ -229,6 +233,8 @@ type Options struct { Hashes string Jarm bool Asn bool + OutputMatchCdn goflags.NormalizedStringSlice + OutputFilterCdn goflags.NormalizedStringSlice } // ParseOptions parses the command line options for application @@ -274,6 +280,7 @@ func ParseOptions() *Options { flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "match response with specified favicon hash (-mfc 1494302000)"), flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), + flagSet.NormalizedStringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", []string{}, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders)), ) createGroup(flagSet, "extractor", "Extractor", @@ -288,6 +295,7 @@ func ParseOptions() *Options { flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "filter response with specified favicon hash (-mfc 1494302000)"), flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), + flagSet.NormalizedStringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", []string{}, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders)), ) createGroup(flagSet, "rate-limit", "Rate-Limit", @@ -478,6 +486,9 @@ func (options *Options) validateOptions() { } } } + if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 { + options.OutputCDN = true + } } // configureOutput configures the output on the screen diff --git a/runner/runner.go b/runner/runner.go index 73c0270..9fc1274 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -591,7 +591,13 @@ func (r *Runner) RunEnumeration() { if len(r.options.matchWordsCount) > 0 && !slice.IntSliceContains(r.options.matchWordsCount, resp.Words) { continue } - + if len(r.options.OutputMatchCdn) > 0 && !stringsutil.EqualFoldAny(resp.CDNName, r.options.OutputMatchCdn...) { + continue + } + if len(r.options.OutputFilterCdn) > 0 && stringsutil.EqualFoldAny(resp.CDNName, r.options.OutputFilterCdn...) { + continue + } + row := resp.str if r.options.JSONOutput { row = resp.JSON(&r.scanopts) From 9113a3be17f01253cc7ae7f30e161a0024117591 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 15 Apr 2022 02:39:01 +0530 Subject: [PATCH 153/355] snapcraft config update --- snapcraft.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/snapcraft.yaml b/snapcraft.yaml index a531459..f15a9b6 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -21,11 +21,11 @@ apps: - removable-media - dot-httpx -plugs: - dot-httpx: - interface: personal-files - write: - - $HOME/.config/httpx +#plugs: +# dot-httpx: +# interface: personal-files +# write: +# - $HOME/.config/httpx parts: httpx: From 06f1c2c4c595123a263baa627b4a7293f551feaf Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 15 Apr 2022 17:41:57 +0200 Subject: [PATCH 154/355] Adding support for custom SNI name --- common/httpx/httpx.go | 1 + common/httpx/option.go | 1 + go.mod | 2 +- go.sum | 3 +++ runner/options.go | 5 ++++- runner/runner.go | 3 ++- 6 files changed, 12 insertions(+), 3 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 23c0461..b33c9a5 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -47,6 +47,7 @@ func New(options *Options) (*HTTPX, error) { if len(options.Resolvers) > 0 { fastdialerOpts.BaseResolvers = options.Resolvers } + fastdialerOpts.SNIName = options.SniName dialer, err := fastdialer.NewDialer(fastdialerOpts) if err != nil { return nil, fmt.Errorf("could not create resolver cache: %s", err) diff --git a/common/httpx/option.go b/common/httpx/option.go index 059665a..b703bd5 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -40,6 +40,7 @@ type Options struct { UnsafeURI string Resolvers []string customCookies []*http.Cookie + SniName string } // DefaultOptions contains the default options diff --git a/go.mod b/go.mod index 561e69d..8d6f329 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/cdncheck v0.0.3 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.15 + github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c diff --git a/go.sum b/go.sum index 0e9450a..bdf1533 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,8 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.15 h1:vOLk8jty+1H9osPRzUT6acD0XBSaglyLS3MlIQaYczk= github.com/projectdiscovery/fastdialer v0.0.15/go.mod h1:Q28lw9oTpiZHq09uFG6YYYLUsUjsOypZ7PXWwQGBB80= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 h1:m2z3BKKqDyzY06pTX8stOWKzZuIWSi6FmwT6wRR052E= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c h1:KDmCXhLLnS/Gc1VDyTxxamRzc8OmHCm1X+f8WQoaTRs= @@ -204,6 +206,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= diff --git a/runner/options.go b/runner/options.go index 509b6bc..b74b610 100644 --- a/runner/options.go +++ b/runner/options.go @@ -5,7 +5,7 @@ import ( "os" "regexp" "strings" - "github.com/projectdiscovery/httpx/common/slice" + "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/goflags" @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/httpx/common/customlist" customport "github.com/projectdiscovery/httpx/common/customports" fileutilz "github.com/projectdiscovery/httpx/common/fileutil" + "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/httpx/common/stringz" ) @@ -229,6 +230,7 @@ type Options struct { Hashes string Jarm bool Asn bool + SniName string } // ParseOptions parses the command line options for application @@ -264,6 +266,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"), flagSet.BoolVar(&options.OutputCDN, "cdn", false, "display cdn in use"), flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), + flagSet.StringVar(&options.SniName, "sni-name", "", "Custom SNI Name"), ) createGroup(flagSet, "matchers", "Matchers", diff --git a/runner/runner.go b/runner/runner.go index 73c0270..3db5e58 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -125,6 +125,7 @@ func New(options *Options) (*Runner, error) { value = strings.TrimSpace(tokens[1]) httpxOptions.CustomHeaders[key] = value } + httpxOptions.SniName = options.SniName runner.hp, err = httpx.New(&httpxOptions) if err != nil { @@ -1293,7 +1294,7 @@ retry: } jarmhash := "" if r.options.Jarm { - jarmhash = hashes.Jarm(fullURL,r.options.Timeout) + jarmhash = hashes.Jarm(fullURL, r.options.Timeout) builder.WriteString(" [") if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(jarmhash).String()) From c32cbbcfa2aa6fd84b12d779900a6d13def3e10f Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 15 Apr 2022 17:45:02 +0200 Subject: [PATCH 155/355] mod tidy --- go.sum | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/go.sum b/go.sum index d06472c..fadea16 100644 --- a/go.sum +++ b/go.sum @@ -122,16 +122,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= -github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g= -github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.15 h1:vOLk8jty+1H9osPRzUT6acD0XBSaglyLS3MlIQaYczk= -github.com/projectdiscovery/fastdialer v0.0.15/go.mod h1:Q28lw9oTpiZHq09uFG6YYYLUsUjsOypZ7PXWwQGBB80= github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 h1:m2z3BKKqDyzY06pTX8stOWKzZuIWSi6FmwT6wRR052E= github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= @@ -206,8 +202,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= From 83f198cf5b4d65e12636cb1940c68a8f47ad4ec6 Mon Sep 17 00:00:00 2001 From: chavacava Date: Mon, 18 Apr 2022 10:49:57 +0200 Subject: [PATCH 156/355] fix #584 fix #584 by passing range value `target` as parameter of the goroutine --- runner/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 73c0270..4392fb9 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -709,7 +709,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. for _, wantedProtocol := range wantedProtocols { for _, method := range scanopts.Methods { wg.Add() - go func(port int, method, protocol string) { + go func(port int, target, method, protocol string) { defer wg.Done() h, _ := urlutil.ChangePort(target, fmt.Sprint(port)) result := r.analyze(hp, protocol, h, method, t, scanopts) @@ -729,7 +729,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. r.process(tt, wg, hp, protocol, scanopts, output) } } - }(port, method, wantedProtocol) + }(port, target, method, wantedProtocol) } } } From 5c4769c40835dc40a2cd5704ceacdf6fd647e62a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Apr 2022 10:04:19 +0000 Subject: [PATCH 157/355] chore(deps): bump golang from 1.18.0-alpine to 1.18.1-alpine Bumps golang from 1.18.0-alpine to 1.18.1-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e2880bc..677c300 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18.0-alpine AS builder +FROM golang:1.18.1-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 5fc82c60f3da105049978d49f734c46776d2e7df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Apr 2022 10:13:37 +0000 Subject: [PATCH 158/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.36 to 0.0.37. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.36...v0.0.37) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 737a37a..4f1232e 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.36 + github.com/projectdiscovery/wappalyzergo v0.0.37 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 8601125..5b581ee 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= -github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g= -github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= @@ -179,8 +177,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.36 h1:8wz4ZvOP4FHMOMn7r4BSDPmQGFAbJk6gNx9w3elsv3A= -github.com/projectdiscovery/wappalyzergo v0.0.36/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.37 h1:hP/Z1B+CGKt+f6yHZRWSHTxaFd9VZ+QM6qcBzohOWoM= +github.com/projectdiscovery/wappalyzergo v0.0.37/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From a83028259b81524f8310b221518fb5ef7ae8b271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel=20Jimeno?= Date: Mon, 18 Apr 2022 13:51:27 +0200 Subject: [PATCH 159/355] runner: save 8 bytes per Result object --- runner/runner.go | 78 ++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 9fc1274..4b019f6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -597,7 +597,7 @@ func (r *Runner) RunEnumeration() { if len(r.options.OutputFilterCdn) > 0 && stringsutil.EqualFoldAny(resp.CDNName, r.options.OutputFilterCdn...) { continue } - + row := resp.str if r.options.JSONOutput { row = resp.JSON(&r.scanopts) @@ -1299,7 +1299,7 @@ retry: } jarmhash := "" if r.options.Jarm { - jarmhash = hashes.Jarm(fullURL,r.options.Timeout) + jarmhash = hashes.Jarm(fullURL, r.options.Timeout) builder.WriteString(" [") if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(jarmhash).String()) @@ -1441,49 +1441,49 @@ func (o AsnResponse) String() string { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` - Request string `json:"request,omitempty" csv:"request"` - ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` - Scheme string `json:"scheme,omitempty" csv:"scheme"` - Port string `json:"port,omitempty" csv:"port"` - Path string `json:"path,omitempty" csv:"path"` - A []string `json:"a,omitempty" csv:"a"` - CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` + ASN interface{} `json:"asn,omitempty" csv:"asn"` + err error + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` + TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty" csv:"tls-grab"` + Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` + CDNName string `json:"cdn-name,omitempty" csv:"cdn-name"` + Port string `json:"port,omitempty" csv:"port"` raw string URL string `json:"url,omitempty" csv:"url"` Input string `json:"input,omitempty" csv:"input"` Location string `json:"location,omitempty" csv:"location"` Title string `json:"title,omitempty" csv:"title"` str string - err error - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"response-body,omitempty" csv:"response-body"` - ContentType string `json:"content-type,omitempty" csv:"content-type"` - Method string `json:"method,omitempty" csv:"method"` - Host string `json:"host,omitempty" csv:"host"` - ContentLength int `json:"content-length,omitempty" csv:"content-length"` - ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` - StatusCode int `json:"status-code,omitempty" csv:"status-code"` - TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty" csv:"tls-grab"` - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - CDNName string `json:"cdn-name,omitempty" csv:"cdn-name"` - ResponseTime string `json:"response-time,omitempty" csv:"response-time"` - Technologies []string `json:"technologies,omitempty" csv:"technologies"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - FinalURL string `json:"final-url,omitempty" csv:"final-url"` - Failed bool `json:"failed" csv:"failed"` - FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` - Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` - ASN interface{} `json:"asn,omitempty" csv:"asn"` - Lines int `json:"lines" csv:"lines"` - Words int `json:"words" csv:"words"` - Jarm string `json:"jarm,omitempty" csv:"jarm"` + Scheme string `json:"scheme,omitempty" csv:"scheme"` + Error string `json:"error,omitempty" csv:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver"` + ResponseBody string `json:"response-body,omitempty" csv:"response-body"` + ContentType string `json:"content-type,omitempty" csv:"content-type"` + Method string `json:"method,omitempty" csv:"method"` + Host string `json:"host,omitempty" csv:"host"` + Path string `json:"path,omitempty" csv:"path"` + FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` + FinalURL string `json:"final-url,omitempty" csv:"final-url"` + ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` + Request string `json:"request,omitempty" csv:"request"` + ResponseTime string `json:"response-time,omitempty" csv:"response-time"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` + ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` + A []string `json:"a,omitempty" csv:"a"` + CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` + Technologies []string `json:"technologies,omitempty" csv:"technologies"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` + Words int `json:"words" csv:"words"` + Lines int `json:"lines" csv:"lines"` + StatusCode int `json:"status-code,omitempty" csv:"status-code"` + ContentLength int `json:"content-length,omitempty" csv:"content-length"` + Failed bool `json:"failed" csv:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` } // JSON the result From da8979569d61bc7759c7067501487df15eef0533 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Mon, 18 Apr 2022 16:06:04 +0300 Subject: [PATCH 160/355] chore: go version update to 1.18 in GitHub actions --- .github/workflows/build-test.yml | 2 +- .github/workflows/functional-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release-binary.yml | 2 +- .github/workflows/sonarcloud.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b17ce23..74ccc30 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index bea5fc4..841226c 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 0ac776b..8f0b292 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 1ab659c..d709397 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -17,7 +17,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v2 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index b64d6ef..e0df696 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -20,7 +20,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: Run unit Tests run: | From d52480f1998b8855d6f578f6f3d7b536a0aa223f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 19 Apr 2022 11:02:31 +0200 Subject: [PATCH 161/355] go mod update --- go.mod | 4 ++-- go.sum | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f98b287..c059123 100644 --- a/go.mod +++ b/go.mod @@ -47,12 +47,12 @@ require github.com/spaolacci/murmur3 v1.1.0 require ( github.com/PuerkitoBio/goquery v1.8.0 + github.com/RumbleDiscovery/jarm-go v0.0.6 + github.com/ammario/ipisp/v2 v2.0.0 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 ) require ( - github.com/RumbleDiscovery/jarm-go v0.0.6 // indirect - github.com/ammario/ipisp/v2 v2.0.0 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 986f755..0d9758a 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,6 @@ github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mo github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/ammario/ipisp v1.0.0 h1:U4xdVMBFWm0/4sHrQ3hVMC+ygg/Ynm4/vdFdkVAex1o= -github.com/ammario/ipisp v1.0.0/go.mod h1:HM60VFpmEWyU+FisnTTHCeswaU3RW0dCVHihgIGUEGM= github.com/ammario/ipisp/v2 v2.0.0 h1:/aRMp5srZViiBfOUGzl/Esqae4s0MDDzm9buhGcZ0XU= github.com/ammario/ipisp/v2 v2.0.0/go.mod h1:bQ6KAL5LnYYEj6olUn+Bzv/im/4Esa5oGkbv9b+uOjo= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= @@ -118,7 +116,6 @@ github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -200,7 +197,6 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= From 4c0b28374e74d825df3668d429c3ae255e23baaa Mon Sep 17 00:00:00 2001 From: sandeep Date: Thu, 21 Apr 2022 13:05:58 +0530 Subject: [PATCH 162/355] misc update --- runner/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 9859875..d2cacbb 100644 --- a/runner/options.go +++ b/runner/options.go @@ -272,7 +272,6 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"), flagSet.BoolVar(&options.OutputCDN, "cdn", false, "display cdn in use"), flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), - flagSet.StringVar(&options.SniName, "sni-name", "", "Custom SNI Name"), ) createGroup(flagSet, "matchers", "Matchers", @@ -334,6 +333,7 @@ func ParseOptions() *Options { flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "list of custom resolver (file or comma separated)"), flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), + flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "Custom TLS SNI name"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"), flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"), From 26f7ed72ae7f02724bc1963a4593e3860cc234aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:11:46 +0000 Subject: [PATCH 163/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.37 to 0.0.38. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.37...v0.0.38) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 541d87e..12820e9 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.37 + github.com/projectdiscovery/wappalyzergo v0.0.38 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 4f85567..b1c48ff 100644 --- a/go.sum +++ b/go.sum @@ -177,8 +177,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.37 h1:hP/Z1B+CGKt+f6yHZRWSHTxaFd9VZ+QM6qcBzohOWoM= -github.com/projectdiscovery/wappalyzergo v0.0.37/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.38 h1:0HGXJfMGMfveM5R0oGIWL0Sc513+gVPoHQkjZxmT2GA= +github.com/projectdiscovery/wappalyzergo v0.0.38/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From e0e2f0b72a039843721679154bfd154df824b343 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:11:50 +0000 Subject: [PATCH 164/355] chore(deps): bump github.com/projectdiscovery/mapcidr Bumps [github.com/projectdiscovery/mapcidr](https://github.com/projectdiscovery/mapcidr) from 0.0.8 to 0.0.9. - [Release notes](https://github.com/projectdiscovery/mapcidr/releases) - [Changelog](https://github.com/projectdiscovery/mapcidr/blob/master/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/mapcidr/compare/v0.0.8...v0.0.9) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/mapcidr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 541d87e..2c9dcde 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 - github.com/projectdiscovery/mapcidr v0.0.8 + github.com/projectdiscovery/mapcidr v0.0.9 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.2 github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea diff --git a/go.sum b/go.sum index 4f85567..47edff9 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,9 @@ github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1: github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo2zlsL4L4E= github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= +github.com/projectdiscovery/mapcidr v0.0.9 h1:PIa09fMHdghlmkUeTgHP9bwYnb3k2wXXM2f6LMj26zg= +github.com/projectdiscovery/mapcidr v0.0.9/go.mod h1:zgsrc+UXwcLcBopUNboiI4tpTICbfdTyJZiBi2tx+NI= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= From 48640ec9cadc662e7f3c6b8396f57af615ff4536 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 28 Apr 2022 14:52:10 +0200 Subject: [PATCH 165/355] Disable static compilation for osx + minor optimizations --- Makefile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 595230d..102b49d 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,17 @@ GOCMD=go GOBUILD=$(GOCMD) build GOMOD=$(GOCMD) mod GOTEST=$(GOCMD) test -GOGET=$(GOCMD) get +GOFLAGS := -v +LDFLAGS := -s -w + +ifneq ($(shell go env GOOS),darwin) +LDFLAGS := -extldflags "-static" +endif all: build build: - $(GOBUILD) -v -ldflags="-extldflags=-static" -o "httpx" cmd/httpx/httpx.go + $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "httpx" cmd/httpx/httpx.go test: - $(GOTEST) -v ./... + $(GOTEST) $(GOFLAGS) ./... tidy: - $(GOMOD) tidy + $(GOMOD) tidy From 9ea89a4a468b3d8ac87eebcbf36978c8f08c75e9 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Fri, 29 Apr 2022 10:24:05 -0500 Subject: [PATCH 166/355] match/filter response time --- go.mod | 31 +++++++-- go.sum | 131 +++++++++++++++++++++++++++++++++++++++ runner/filteroperator.go | 45 ++++++++++++++ runner/options.go | 7 +++ runner/runner.go | 66 +++++++++++++++++++- 5 files changed, 273 insertions(+), 7 deletions(-) create mode 100644 runner/filteroperator.go diff --git a/go.mod b/go.mod index ad9e76b..9932f1d 100644 --- a/go.mod +++ b/go.mod @@ -11,16 +11,16 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.18 - github.com/miekg/dns v1.1.46 // indirect + github.com/miekg/dns v1.1.48 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c + github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.0.7 + github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 @@ -38,8 +38,8 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 - golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/text v0.3.7 ) @@ -53,17 +53,31 @@ require ( ) require ( + git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect + github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/caddyserver/certmagic v0.16.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/goburrow/cache v0.1.4 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/karrick/godirwalk v1.16.1 // indirect + github.com/klauspost/compress v1.15.1 // indirect + github.com/klauspost/cpuid/v2 v2.0.11 // indirect + github.com/libdns/libdns v0.2.1 // indirect + github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect + github.com/mholt/acmez v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect + github.com/projectdiscovery/interactsh v1.0.3 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.13 // indirect @@ -73,10 +87,15 @@ require ( github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect - golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.21.0 // indirect + goftp.io/server/v2 v2.0.0 // indirect + golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index c8ec4a6..67ae1f4 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,13 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= +git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY9efxTKK5aFic5C5KybqQelGcX+JdM69KoTo= +github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd/go.mod h1:AqtPw7WNT0O69k+AbPKWVGYeW94TqgMW/g+Ppc8AZr4= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= @@ -15,10 +22,19 @@ github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9or github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= +github.com/caddyserver/certmagic v0.16.0 h1:nM6Fm+OSnTpx/uRWmN++I2fYq006uhi6m6m3rD1Jjtg= +github.com/caddyserver/certmagic v0.16.0/go.mod h1:jKQ5n+ViHAr6DbPwEGLTSM2vDwTO6EvCKBblBRUvvuQ= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -30,11 +46,23 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= +github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -42,24 +70,33 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -70,6 +107,11 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid/v2 v2.0.11 h1:i2lw1Pm7Yi/4O6XCSyJWqEHI2MDw2FzUK6o/D21xn2A= +github.com/klauspost/cpuid/v2 v2.0.11/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -79,11 +121,17 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= +github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 h1:wIONC+HMNRqmWBjuMxhatuSzHaljStc4gjDeKycxy0A= +github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3/go.mod h1:37YR9jabpiIxsb8X9VCIx8qFOjTDIIrIHHODa8C4gz0= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= +github.com/mholt/acmez v1.0.2 h1:C8wsEBIUVi6e0DYoxqCcFuXtwc4AWXL/jgcDjF7mjVo= +github.com/mholt/acmez v1.0.2/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ceceM= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -91,6 +139,10 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= +github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -116,6 +168,7 @@ github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -134,10 +187,14 @@ github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0 github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c h1:KDmCXhLLnS/Gc1VDyTxxamRzc8OmHCm1X+f8WQoaTRs= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc h1:dbDgsj26PW06L3zMo7AT08IbEqMd2u8QQ1BvlfMAY2w= +github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 h1:M9DJMe3zMnvHkt5McU1IpoTQbBYc2aB1nsOBLqWlQOs= +github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -146,6 +203,8 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUA github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= +github.com/projectdiscovery/interactsh v1.0.3 h1:QFL/fFTfvhLeIRFBSguWP8r6h8vf65Bhkf8wzKU2/qQ= +github.com/projectdiscovery/interactsh v1.0.3/go.mod h1:3943djLJ4SZBZfJ7A2XEy8xspfFmT0BDiN9FCW7PAXY= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 h1:VZ9H51A7tI7R/9I5W5l960Nkq7eMJqGd3y1wsuwzdjE= @@ -180,18 +239,23 @@ github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmON github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= github.com/projectdiscovery/wappalyzergo v0.0.38 h1:0HGXJfMGMfveM5R0oGIWL0Sc513+gVPoHQkjZxmT2GA= github.com/projectdiscovery/wappalyzergo v0.0.38/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -224,31 +288,61 @@ github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +goftp.io/server/v2 v2.0.0 h1:FF8JKXXKDxAeO1uXEZz7G+IZwCDhl19dpVIlDtp3QAg= +goftp.io/server/v2 v2.0.0/go.mod h1:7+H/EIq7tXdfo1Muu5p+l3oQ6rYkDZ8lY7IM5d5kVdQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= +golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -260,16 +354,25 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -292,18 +395,29 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -311,6 +425,16 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -319,6 +443,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -326,16 +451,22 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/runner/filteroperator.go b/runner/filteroperator.go new file mode 100644 index 0000000..0eda27f --- /dev/null +++ b/runner/filteroperator.go @@ -0,0 +1,45 @@ +package runner + +import ( + "fmt" + "strconv" + "strings" +) + +var ( + greaterThan = ">" + lessThan = "<" + equal = "=" + greaterThanEq = ">=" + lessThanEq = "<=" + notEq = "!=" + compareOperators = []string{greaterThanEq, lessThanEq, equal, lessThan, greaterThan, notEq} +) + +type FilterOperator struct { + flag string +} + +// Parse the given value into operator and value pair +func (f FilterOperator) Parse(flagValue string) (string, float64, error) { + var ( + operator string + value float64 + err error + ) + for _, op := range compareOperators { + if strings.Contains(flagValue, op) { + spl := strings.SplitAfter(flagValue, op) + operator = spl[0] + value, err = strconv.ParseFloat(spl[1], 64) + if err != nil { + return operator, value, fmt.Errorf("invalid value provided for %s", f.flag) + } + break + } + } + if operator == "" { + return operator, value, fmt.Errorf("invalid operator provided for %s, valid operators are %s", f.flag, strings.Join(compareOperators, ",")) + } + return operator, value, nil +} diff --git a/runner/options.go b/runner/options.go index d2cacbb..cbf7c4b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -237,6 +237,8 @@ type Options struct { OutputMatchCdn goflags.NormalizedStringSlice OutputFilterCdn goflags.NormalizedStringSlice SniName string + OutputMatchResponseTime string + OutputFilterResponseTime string } // ParseOptions parses the command line options for application @@ -283,6 +285,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), flagSet.NormalizedStringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", []string{}, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders)), + flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time"), ) createGroup(flagSet, "extractor", "Extractor", @@ -298,6 +301,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), flagSet.NormalizedStringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", []string{}, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders)), + flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", @@ -509,6 +513,9 @@ func (options *Options) configureOutput() { if options.Silent { gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) } + if len(options.OutputMatchResponseTime) > 0 || len(options.OutputFilterResponseTime) > 0 { + options.OutputResponseTime = true + } } func (options *Options) configureResume() error { diff --git a/runner/runner.go b/runner/runner.go index 9e4c6cc..04b3ae4 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -598,7 +598,71 @@ func (r *Runner) RunEnumeration() { if len(r.options.OutputFilterCdn) > 0 && stringsutil.EqualFoldAny(resp.CDNName, r.options.OutputFilterCdn...) { continue } - + + if r.options.OutputMatchResponseTime != "" { + filterOps := FilterOperator{flag: "-mrt, -match-response-time"} + operator, value, err := filterOps.Parse(r.options.OutputMatchResponseTime) + if err != nil { + gologger.Fatal().Msg(err.Error()) + } + respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) + switch operator { + // take negation of >= and > + case greaterThanEq, greaterThan: + if respTimeTaken.Seconds() < value { + continue + } + // take negation of <= and < + case lessThanEq, lessThan: + if respTimeTaken.Seconds() > value { + continue + } + // take negation of = + case equal: + if respTimeTaken.Seconds() != value { + continue + } + // take negation of != + case notEq: + if respTimeTaken.Seconds() == value { + continue + } + } + } + if r.options.OutputFilterResponseTime != "" { + filterOps := FilterOperator{flag: "-frt, -filter-response-time"} + operator, value, err := filterOps.Parse(r.options.OutputFilterResponseTime) + if err != nil { + gologger.Fatal().Msg(err.Error()) + } + respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) + switch operator { + case greaterThanEq: + if respTimeTaken.Seconds() >= value { + continue + } + case lessThanEq: + if respTimeTaken.Seconds() <= value { + continue + } + case equal: + if respTimeTaken.Seconds() == value { + continue + } + case lessThan: + if respTimeTaken.Seconds() < value { + continue + } + case greaterThan: + if respTimeTaken.Seconds() > value { + continue + } + case notEq: + if respTimeTaken.Seconds() != value { + continue + } + } + } row := resp.str if r.options.JSONOutput { row = resp.JSON(&r.scanopts) From 1cd05e10710dedbd77a058fa195ce57b67992549 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Fri, 29 Apr 2022 10:48:28 -0500 Subject: [PATCH 167/355] readme updates --- README.md | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2c20964..bd7de68 100644 --- a/README.md +++ b/README.md @@ -109,27 +109,29 @@ PROBES: -probe display probe status MATCHERS: - -mc, -match-code string match response with specified status code (-mc 200,302) - -ml, -match-length string match response with specified content length (-ml 100,102) - -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) - -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) - -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) - -ms, -match-string string match response with specified string (-ms admin) - -mr, -match-regex string match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (-mcdn azure,google) + -mc, -match-code string match response with specified status code (-mc 200,302) + -ml, -match-length string match response with specified content length (-ml 100,102) + -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) + -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) + -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) + -ms, -match-string string match response with specified string (-ms admin) + -mr, -match-regex string match response with specified regex (-mr admin) + -mcdn, -match-cdn string[] match host with specified cdn provider (-mcdn azure,google) + -mrt, -match-response-time string match response with specified response time(-mrt ">=3") EXTRACTOR: -er, -extract-regex string display response content for specified regex FILTERS: - -fc, -filter-code string filter response with specified status code (-fc 403,401) - -fl, -filter-length string filter response with specified content length (-fl 23,33) - -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) - -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) - -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) - -fs, -filter-string string filter response with specified string (-fs admin) - -fe, -filter-regex string filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (-fcdn oracle,cloudflare) + -fc, -filter-code string filter response with specified status code (-fc 403,401) + -fl, -filter-length string filter response with specified content length (-fl 23,33) + -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) + -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) + -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) + -fs, -filter-string string filter response with specified string (-fs admin) + -fe, -filter-regex string filter response with specified regex (-fe admin) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (-fcdn oracle,cloudflare) + -frt, -filter-response-time string filter response with specified response time(-frt "<=4") RATE-LIMIT: -t, -threads int number of threads to use (default 50) From 80b16a598a904ffcfbe79a4e5ef3b7d6736d8845 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 10:06:09 +0000 Subject: [PATCH 168/355] chore(deps): bump github/codeql-action from 1 to 2 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 1 to 2. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v1...v2) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7c34372..2bbccfd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,12 +28,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 \ No newline at end of file + uses: github/codeql-action/analyze@v2 \ No newline at end of file From 51d2f5ad6418eb54303db3910a05fd2060fd18f6 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 3 May 2022 13:56:16 +0530 Subject: [PATCH 169/355] go mod update --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ad9e76b..2afe4cb 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.38 + github.com/projectdiscovery/wappalyzergo v0.0.40 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index c8ec4a6..d10a397 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,8 @@ github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmON github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= github.com/projectdiscovery/wappalyzergo v0.0.38 h1:0HGXJfMGMfveM5R0oGIWL0Sc513+gVPoHQkjZxmT2GA= github.com/projectdiscovery/wappalyzergo v0.0.38/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.40 h1:E/rADqztV3+APjE5u/SEH046Ylc1R2wzG7TZVATf0is= +github.com/projectdiscovery/wappalyzergo v0.0.40/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 032c535292431b2ab50b186bfb6caf69c5e23fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel=20Jimeno?= Date: Tue, 3 May 2022 10:02:35 +0200 Subject: [PATCH 170/355] httpx: consider CDN check error as non-fatal Fixes #599 --- common/httpx/httpx.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index b33c9a5..27bb645 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -16,6 +16,7 @@ import ( "github.com/microcosm-cc/bluemonday" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" pdhttputil "github.com/projectdiscovery/httputil" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" @@ -142,7 +143,7 @@ func New(options *Options) (*HTTPX, error) { if options.CdnCheck || options.ExcludeCdn { httpx.cdn, err = cdncheck.NewWithCache() if err != nil { - return nil, fmt.Errorf("could not create cdn check: %s", err) + gologger.Error().Msgf("could not create cdn check: %v", err) } } From 1fdce482d5b4e7b84b99e8945211c30f7498a360 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 3 May 2022 10:35:00 -0500 Subject: [PATCH 171/355] included time unit --- runner/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/options.go b/runner/options.go index cbf7c4b..287c06f 100644 --- a/runner/options.go +++ b/runner/options.go @@ -285,7 +285,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), flagSet.NormalizedStringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", []string{}, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders)), - flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time"), + flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time(time unit- second)"), ) createGroup(flagSet, "extractor", "Extractor", @@ -301,7 +301,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), flagSet.NormalizedStringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", []string{}, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders)), - flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time"), + flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time(time unit- second)"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", From bca081f35cd89248a8b4391439b9d331dddbccf8 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 3 May 2022 21:01:43 -0500 Subject: [PATCH 172/355] space separated arguments for cli flag --- runner/filteroperator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/filteroperator.go b/runner/filteroperator.go index 0eda27f..ec33cb4 100644 --- a/runner/filteroperator.go +++ b/runner/filteroperator.go @@ -30,8 +30,8 @@ func (f FilterOperator) Parse(flagValue string) (string, float64, error) { for _, op := range compareOperators { if strings.Contains(flagValue, op) { spl := strings.SplitAfter(flagValue, op) - operator = spl[0] - value, err = strconv.ParseFloat(spl[1], 64) + operator = strings.Trim(spl[0], " ") + value, err = strconv.ParseFloat(strings.Trim(spl[1], " "), 64) if err != nil { return operator, value, fmt.Errorf("invalid value provided for %s", f.flag) } From a0e5b0c1693d8cdd516a9bf93fc900cf72dfe58f Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 6 May 2022 13:45:09 +0530 Subject: [PATCH 173/355] misc updates --- README.md | 4 ++-- runner/options.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bd7de68..872ba2d 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ MATCHERS: -ms, -match-string string match response with specified string (-ms admin) -mr, -match-regex string match response with specified regex (-mr admin) -mcdn, -match-cdn string[] match host with specified cdn provider (-mcdn azure,google) - -mrt, -match-response-time string match response with specified response time(-mrt ">=3") + -mrt, -match-response-time string match response with specified response time in seconds (-mrt '> 1') EXTRACTOR: -er, -extract-regex string display response content for specified regex @@ -131,7 +131,7 @@ FILTERS: -fs, -filter-string string filter response with specified string (-fs admin) -fe, -filter-regex string filter response with specified regex (-fe admin) -fcdn, -filter-cdn string[] filter host with specified cdn provider (-fcdn oracle,cloudflare) - -frt, -filter-response-time string filter response with specified response time(-frt "<=4") + -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') RATE-LIMIT: -t, -threads int number of threads to use (default 50) diff --git a/runner/options.go b/runner/options.go index 287c06f..25407e3 100644 --- a/runner/options.go +++ b/runner/options.go @@ -285,7 +285,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), flagSet.NormalizedStringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", []string{}, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders)), - flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time(time unit- second)"), + flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"), ) createGroup(flagSet, "extractor", "Extractor", @@ -301,7 +301,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), flagSet.NormalizedStringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", []string{}, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders)), - flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time(time unit- second)"), + flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", From 0ae5773318a9a90a689d20d3aa109a66587d6daa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 10:09:17 +0000 Subject: [PATCH 174/355] chore(deps): bump docker/setup-buildx-action from 1 to 2 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index f253b3d..ebc3d23 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -23,7 +23,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v1 From c6363bdd36d97d1504462b2e8064b8e573d379de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 10:09:19 +0000 Subject: [PATCH 175/355] chore(deps): bump docker/setup-qemu-action from 1 to 2 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index f253b3d..939b4cc 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -20,7 +20,7 @@ jobs: echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/httpx/releases/latest" | jq -r .tag_name)" - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 From 48fa60fce3b6ffbe038c6aa638c99ba6d4ce579c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 10:09:23 +0000 Subject: [PATCH 176/355] chore(deps): bump docker/build-push-action from 2 to 3 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 2 to 3. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index f253b3d..8de92f5 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -32,7 +32,7 @@ jobs: password: ${{ secrets.DOCKER_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64,linux/arm From b8da366062c1b50a87a797c2d168353ab79b746c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 10:17:26 +0000 Subject: [PATCH 177/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.40 to 0.0.41. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.40...v0.0.41) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 19 +-------- go.sum | 133 +++------------------------------------------------------ 2 files changed, 7 insertions(+), 145 deletions(-) diff --git a/go.mod b/go.mod index 04bc6e5..103c0bf 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.40 + github.com/projectdiscovery/wappalyzergo v0.0.41 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect @@ -53,31 +53,18 @@ require ( ) require ( - git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a // indirect - github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/caddyserver/certmagic v0.16.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/goburrow/cache v0.1.4 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/karrick/godirwalk v1.16.1 // indirect - github.com/klauspost/compress v1.15.1 // indirect - github.com/klauspost/cpuid/v2 v2.0.11 // indirect - github.com/libdns/libdns v0.2.1 // indirect - github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 // indirect - github.com/mholt/acmez v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect - github.com/projectdiscovery/interactsh v1.0.3 // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.13 // indirect @@ -87,14 +74,10 @@ require ( github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.21.0 // indirect - goftp.io/server/v2 v2.0.0 // indirect golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/protobuf v1.27.1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect diff --git a/go.sum b/go.sum index 6d2c189..50af12d 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,6 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= -git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= -github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd h1:RTWs+wEY9efxTKK5aFic5C5KybqQelGcX+JdM69KoTo= -github.com/Mzack9999/ldapserver v1.0.2-0.20211229000134-b44a0d6ad0dd/go.mod h1:AqtPw7WNT0O69k+AbPKWVGYeW94TqgMW/g+Ppc8AZr4= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= @@ -22,19 +15,12 @@ github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9or github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= -github.com/caddyserver/certmagic v0.16.0 h1:nM6Fm+OSnTpx/uRWmN++I2fYq006uhi6m6m3rD1Jjtg= -github.com/caddyserver/certmagic v0.16.0/go.mod h1:jKQ5n+ViHAr6DbPwEGLTSM2vDwTO6EvCKBblBRUvvuQ= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= @@ -46,23 +32,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= -github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -70,33 +44,24 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -107,11 +72,6 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid/v2 v2.0.11 h1:i2lw1Pm7Yi/4O6XCSyJWqEHI2MDw2FzUK6o/D21xn2A= -github.com/klauspost/cpuid/v2 v2.0.11/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -121,28 +81,19 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= -github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 h1:wIONC+HMNRqmWBjuMxhatuSzHaljStc4gjDeKycxy0A= -github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3/go.mod h1:37YR9jabpiIxsb8X9VCIx8qFOjTDIIrIHHODa8C4gz0= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= -github.com/mholt/acmez v1.0.2 h1:C8wsEBIUVi6e0DYoxqCcFuXtwc4AWXL/jgcDjF7mjVo= -github.com/mholt/acmez v1.0.2/go.mod h1:8qnn8QA/Ewx8E3ZSsmscqsIjhhpxuy9vqdgbX2ceceM= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= -github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -168,7 +119,6 @@ github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -185,13 +135,12 @@ github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 h1: github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= -github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c h1:KDmCXhLLnS/Gc1VDyTxxamRzc8OmHCm1X+f8WQoaTRs= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc h1:dbDgsj26PW06L3zMo7AT08IbEqMd2u8QQ1BvlfMAY2w= github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 h1:M9DJMe3zMnvHkt5McU1IpoTQbBYc2aB1nsOBLqWlQOs= github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= @@ -203,8 +152,6 @@ github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUA github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= -github.com/projectdiscovery/interactsh v1.0.3 h1:QFL/fFTfvhLeIRFBSguWP8r6h8vf65Bhkf8wzKU2/qQ= -github.com/projectdiscovery/interactsh v1.0.3/go.mod h1:3943djLJ4SZBZfJ7A2XEy8xspfFmT0BDiN9FCW7PAXY= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 h1:VZ9H51A7tI7R/9I5W5l960Nkq7eMJqGd3y1wsuwzdjE= @@ -237,27 +184,20 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.38 h1:0HGXJfMGMfveM5R0oGIWL0Sc513+gVPoHQkjZxmT2GA= -github.com/projectdiscovery/wappalyzergo v0.0.38/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/projectdiscovery/wappalyzergo v0.0.40 h1:E/rADqztV3+APjE5u/SEH046Ylc1R2wzG7TZVATf0is= -github.com/projectdiscovery/wappalyzergo v0.0.40/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.41 h1:t6zsfLiMFyjsIv0BYYKSDSxDbsG+FEuO1/TjE/eD55o= +github.com/projectdiscovery/wappalyzergo v0.0.41/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -290,61 +230,32 @@ github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -goftp.io/server/v2 v2.0.0 h1:FF8JKXXKDxAeO1uXEZz7G+IZwCDhl19dpVIlDtp3QAg= -goftp.io/server/v2 v2.0.0/go.mod h1:7+H/EIq7tXdfo1Muu5p+l3oQ6rYkDZ8lY7IM5d5kVdQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -354,27 +265,20 @@ golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -394,32 +298,24 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -427,16 +323,6 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -445,7 +331,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -453,22 +338,16 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From db3a27a1ac74b3efc944a337301109b5cd8f03ba Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 9 May 2022 15:01:38 -0500 Subject: [PATCH 178/355] frt/mrt improvemets --- cmd/integration-test/http.go | 12 +++++++++++- runner/filteroperator.go | 17 ++++++++++------- runner/runner.go | 23 +++++++++++------------ 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index e7b930e..bd0f6d8 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -25,11 +25,16 @@ var httpTestcases = map[string]testutils.TestCase{ "Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path "Regression test for: https://github.com/projectdiscovery/httpx/issues/433": &issue433{}, // new line scanning with title flag "Request URI to existing file - https://github.com/projectdiscovery/httpx/issues/480": &issue480{}, // request uri pointing to existing file + "Standard HTTP GET Request with match response time": &standardHttpGet{mrt: true, inputValue: "\"<10s\""}, + "Standard HTTP GET Request with filter response time": &standardHttpGet{frt: true, inputValue: "\">3ms\""}, } type standardHttpGet struct { tls bool unsafe bool + mrt bool + frt bool + inputValue string stdinPath string path string expectedOutput string @@ -55,7 +60,12 @@ func (h *standardHttpGet) Execute() error { if h.path != "" { extra = append(extra, "-path", "\""+h.path+"\"") } - + if h.mrt { + extra = append(extra, "-mrt", h.inputValue) + } + if h.frt { + extra = append(extra, "-frt", h.inputValue) + } URL := ts.URL if h.stdinPath != "" { URL += h.stdinPath diff --git a/runner/filteroperator.go b/runner/filteroperator.go index ec33cb4..69f93cc 100644 --- a/runner/filteroperator.go +++ b/runner/filteroperator.go @@ -2,8 +2,8 @@ package runner import ( "fmt" - "strconv" "strings" + "time" ) var ( @@ -21,18 +21,21 @@ type FilterOperator struct { } // Parse the given value into operator and value pair -func (f FilterOperator) Parse(flagValue string) (string, float64, error) { +func (f FilterOperator) Parse(flagValue string) (string, time.Duration, error) { var ( operator string - value float64 + value time.Duration err error ) for _, op := range compareOperators { if strings.Contains(flagValue, op) { - spl := strings.SplitAfter(flagValue, op) - operator = strings.Trim(spl[0], " ") - value, err = strconv.ParseFloat(strings.Trim(spl[1], " "), 64) - if err != nil { + splittedFlagValue := strings.SplitAfter(flagValue, op) + operator = strings.Trim(splittedFlagValue[0], " ") + timeVal := strings.Trim(splittedFlagValue[1], " ") + value, err = time.ParseDuration(timeVal) + if err != nil && strings.Contains(err.Error(), "missing unit") { + value, _ = time.ParseDuration(fmt.Sprintf("%ss", timeVal)) + } else if err != nil { return operator, value, fmt.Errorf("invalid value provided for %s", f.flag) } break diff --git a/runner/runner.go b/runner/runner.go index 04b3ae4..6f1152f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -597,8 +597,7 @@ func (r *Runner) RunEnumeration() { } if len(r.options.OutputFilterCdn) > 0 && stringsutil.EqualFoldAny(resp.CDNName, r.options.OutputFilterCdn...) { continue - } - + } if r.options.OutputMatchResponseTime != "" { filterOps := FilterOperator{flag: "-mrt, -match-response-time"} operator, value, err := filterOps.Parse(r.options.OutputMatchResponseTime) @@ -609,22 +608,22 @@ func (r *Runner) RunEnumeration() { switch operator { // take negation of >= and > case greaterThanEq, greaterThan: - if respTimeTaken.Seconds() < value { + if respTimeTaken < value { continue } // take negation of <= and < case lessThanEq, lessThan: - if respTimeTaken.Seconds() > value { + if respTimeTaken > value { continue } // take negation of = case equal: - if respTimeTaken.Seconds() != value { + if respTimeTaken != value { continue } // take negation of != case notEq: - if respTimeTaken.Seconds() == value { + if respTimeTaken == value { continue } } @@ -638,27 +637,27 @@ func (r *Runner) RunEnumeration() { respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) switch operator { case greaterThanEq: - if respTimeTaken.Seconds() >= value { + if respTimeTaken >= value { continue } case lessThanEq: - if respTimeTaken.Seconds() <= value { + if respTimeTaken <= value { continue } case equal: - if respTimeTaken.Seconds() == value { + if respTimeTaken == value { continue } case lessThan: - if respTimeTaken.Seconds() < value { + if respTimeTaken < value { continue } case greaterThan: - if respTimeTaken.Seconds() > value { + if respTimeTaken > value { continue } case notEq: - if respTimeTaken.Seconds() != value { + if respTimeTaken != value { continue } } From ca74d8f575d476b54540b52e885bb481e3d2eca5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 May 2022 21:54:48 +0000 Subject: [PATCH 179/355] chore(deps): bump docker/login-action from 1 to 2 Bumps [docker/login-action](https://github.com/docker/login-action) from 1 to 2. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 7c2aee5..72c1cfa 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -26,7 +26,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} From f0430675d8a3bb8373dcd25e519fadde75172ffa Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 10 May 2022 19:43:58 +0200 Subject: [PATCH 180/355] Fixing SNI for HTTP transport --- common/httpx/httpx.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 27bb645..16e3360 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -113,6 +113,9 @@ func New(options *Options) (*HTTPX, error) { }, DisableKeepAlives: true, } + if httpx.Options.SniName != "" { + transport.TLSClientConfig.ServerName = httpx.Options.SniName + } if httpx.Options.HTTPProxy != "" { proxyURL, parseErr := url.Parse(httpx.Options.HTTPProxy) @@ -128,14 +131,18 @@ func New(options *Options) (*HTTPX, error) { CheckRedirect: redirectFunc, }, retryablehttpOptions) - httpx.client2 = &http.Client{ - Transport: &http2.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - AllowHTTP: true, + transport2 := &http2.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, }, - Timeout: httpx.Options.Timeout, + AllowHTTP: true, + } + if httpx.Options.SniName != "" { + transport2.TLSClientConfig.ServerName = httpx.Options.SniName + } + httpx.client2 = &http.Client{ + Transport: transport2, + Timeout: httpx.Options.Timeout, } httpx.htmlPolicy = bluemonday.NewPolicy() From 21e41c6c6674c65efa02b47979ad9c52945f5689 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 12 May 2022 14:50:52 -0500 Subject: [PATCH 181/355] added ascii hostname support --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 103c0bf..7ed023d 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 + github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933 github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c @@ -38,7 +38,7 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/text v0.3.7 ) diff --git a/go.sum b/go.sum index 50af12d..b60e09b 100644 --- a/go.sum +++ b/go.sum @@ -133,6 +133,8 @@ github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 h1:m2z3BKKqDyzY06pTX8stOWKzZuIWSi6FmwT6wRR052E= github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933 h1:0roPUQaDmZOXe2Ua15jzGMVMCH8Cd32us9LcF73H+xs= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= @@ -269,6 +271,8 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 2b1d4707211b65b699edf8c06eee3ef9ea702c21 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Fri, 13 May 2022 09:07:30 -0500 Subject: [PATCH 182/355] dependency update --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7ed023d..1121684 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933 + github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c diff --git a/go.sum b/go.sum index b60e09b..b0e9a0c 100644 --- a/go.sum +++ b/go.sum @@ -135,6 +135,8 @@ github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 h1: github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933 h1:0roPUQaDmZOXe2Ua15jzGMVMCH8Cd32us9LcF73H+xs= github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf h1:Vx4wbfI9FShdCUU3XsfyNAZwfm8AAvDFxtop8EZEhRw= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= From 7451dd7c156df15a3e06619033b4887f06f6f32a Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 15 May 2022 21:08:20 +0200 Subject: [PATCH 183/355] Extending ASCII conversion support --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1121684..5b280ef 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf + github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c diff --git a/go.sum b/go.sum index b0e9a0c..34bca2d 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,10 @@ github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933 h1: github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf h1:Vx4wbfI9FShdCUU3XsfyNAZwfm8AAvDFxtop8EZEhRw= github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220515110130-dde30ef8efb2 h1:8fsALX3kp7c0ymxS+w+nT20UCuWia7+JVueQ7Alygho= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220515110130-dde30ef8efb2/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 h1:uxOMKHOajVaiz/pMUzc3nvWhICa/euZN+SoVzpqcPiE= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= From bc7be66905e0be1bd6a3c71eef355302c0deb692 Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 15 May 2022 21:08:33 +0200 Subject: [PATCH 184/355] adding vscode folder to ignore list --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2390b4e..b8f8c16 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ cmd/functional-test/httpx_dev cmd/functional-test/functional-test cmd/functional-test/httpx cmd/functional-test/*.cfg - +.vscode/ From 396c28efcf04f758e5d5b83979615426b51fbf72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 10:07:50 +0000 Subject: [PATCH 185/355] chore(deps): bump golang from 1.18.1-alpine to 1.18.2-alpine Bumps golang from 1.18.1-alpine to 1.18.2-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 677c300..233d5eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18.1-alpine AS builder +FROM golang:1.18.2-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 550dbf7f5c68136ac8767d8a3b99835ec29bcbca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 10:07:51 +0000 Subject: [PATCH 186/355] chore(deps): bump golangci/golangci-lint-action from 3.1.0 to 3.2.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.1.0...v3.2.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 8f0b292..d8b1dbc 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -16,7 +16,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.1.0 + uses: golangci/golangci-lint-action@v3.2.0 with: version: latest args: --timeout 5m From 1616e8506b5215a7647fe34a74a9dca97d01a0e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 May 2022 10:14:05 +0000 Subject: [PATCH 187/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.41 to 0.0.42. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.41...v0.0.42) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 1121684..b4e5def 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.41 + github.com/projectdiscovery/wappalyzergo v0.0.42 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index b0e9a0c..7c461ba 100644 --- a/go.sum +++ b/go.sum @@ -131,10 +131,6 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3 h1:m2z3BKKqDyzY06pTX8stOWKzZuIWSi6FmwT6wRR052E= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220415153642-29a3e094f5d3/go.mod h1:wn6jSJ1fIO6kLplFEbFIkRB6Kj/Q6VngnzKuBHLVPiI= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933 h1:0roPUQaDmZOXe2Ua15jzGMVMCH8Cd32us9LcF73H+xs= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220512171112-e6cca11bc933/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf h1:Vx4wbfI9FShdCUU3XsfyNAZwfm8AAvDFxtop8EZEhRw= github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= @@ -188,8 +184,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.41 h1:t6zsfLiMFyjsIv0BYYKSDSxDbsG+FEuO1/TjE/eD55o= -github.com/projectdiscovery/wappalyzergo v0.0.41/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.42 h1:4DuJVpgesHSnUNVG+l62QIYEmcMXu3PLyhS1Wp75c30= +github.com/projectdiscovery/wappalyzergo v0.0.42/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -271,8 +267,6 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -306,7 +300,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= From f75729c1b91c933be5244780b3fc347a79f01874 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Tue, 17 May 2022 11:20:35 -0500 Subject: [PATCH 188/355] cname with ports flag bug-fix (#625) --- runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 6f1152f..6c08c16 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1230,7 +1230,7 @@ retry: builder.WriteString(fmt.Sprintf(" [%s]", ip)) } - ips, cnames, err := getDNSData(hp, domain) + ips, cnames, err := getDNSData(hp, URL.Host) if err != nil { ips = append(ips, ip) } From b43b9d7c6d3f4d3040d7faefdcf316a15198ca41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 May 2022 15:55:41 +0530 Subject: [PATCH 189/355] chore(deps): bump github.com/corpix/uarand from 0.1.1 to 0.2.0 (#628) Bumps [github.com/corpix/uarand](https://github.com/corpix/uarand) from 0.1.1 to 0.2.0. - [Release notes](https://github.com/corpix/uarand/releases) - [Commits](https://github.com/corpix/uarand/compare/v0.1.1...v0.2.0) --- updated-dependencies: - dependency-name: github.com/corpix/uarand dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 14 ++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index fd05691..361c466 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/akrylysov/pogreb v0.10.1 // indirect github.com/bluele/gcache v0.0.2 - github.com/corpix/uarand v0.1.1 + github.com/corpix/uarand v0.2.0 github.com/golang/snappy v0.0.4 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 diff --git a/go.sum b/go.sum index 58c08c4..8d67df5 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,3 @@ -github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= -github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= @@ -23,9 +20,8 @@ github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= -github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= -github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= -github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= +github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= +github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -94,7 +90,6 @@ github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0 github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -103,7 +98,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= -github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -131,10 +125,6 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf h1:Vx4wbfI9FShdCUU3XsfyNAZwfm8AAvDFxtop8EZEhRw= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220512180107-5f9302406cdf/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220515110130-dde30ef8efb2 h1:8fsALX3kp7c0ymxS+w+nT20UCuWia7+JVueQ7Alygho= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220515110130-dde30ef8efb2/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 h1:uxOMKHOajVaiz/pMUzc3nvWhICa/euZN+SoVzpqcPiE= github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= From daf4ad6c5ae1bdf45ca2609954d1f2806528a069 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 19 May 2022 11:29:07 +0200 Subject: [PATCH 190/355] Fixing all ips probe with ports (#626) * Fixing all ips probe with standard ports * using custom type for targets --- common/httpx/types.go | 8 ++++ runner/runner.go | 88 +++++++++++++++++++++---------------------- 2 files changed, 51 insertions(+), 45 deletions(-) create mode 100644 common/httpx/types.go diff --git a/common/httpx/types.go b/common/httpx/types.go new file mode 100644 index 0000000..7683282 --- /dev/null +++ b/common/httpx/types.go @@ -0,0 +1,8 @@ +package httpx + +// Target of the scan with ip|host header customization +type Target struct { + Host string + CustomHost string + CustomIP string +} diff --git a/runner/runner.go b/runner/runner.go index 6c08c16..e7be198 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -597,7 +597,7 @@ func (r *Runner) RunEnumeration() { } if len(r.options.OutputFilterCdn) > 0 && stringsutil.EqualFoldAny(resp.CDNName, r.options.OutputFilterCdn...) { continue - } + } if r.options.OutputMatchResponseTime != "" { filterOps := FilterOperator{flag: "-mrt, -match-response-time"} operator, value, err := filterOps.Parse(r.options.OutputMatchResponseTime) @@ -738,7 +738,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. for _, method := range scanopts.Methods { for _, prot := range protocols { wg.Add() - go func(target, method, protocol string) { + go func(target httpx.Target, method, protocol string) { defer wg.Done() result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result @@ -779,10 +779,10 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. for _, wantedProtocol := range wantedProtocols { for _, method := range scanopts.Methods { wg.Add() - go func(port int, target, method, protocol string) { + go func(port int, target httpx.Target, method, protocol string) { defer wg.Done() - h, _ := urlutil.ChangePort(target, fmt.Sprint(port)) - result := r.analyze(hp, protocol, h, method, t, scanopts) + target.Host, _ = urlutil.ChangePort(target.Host, fmt.Sprint(port)) + result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false @@ -810,8 +810,8 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. } // returns all the targets within a cidr range or the single target -func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { - results := make(chan string) +func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { + results := make(chan httpx.Target) go func() { defer close(results) @@ -819,7 +819,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { // * // spaces if strings.ContainsAny(target, "*") || strings.HasPrefix(target, ".") { - // trim * and/or . (prefix) from the target to return the domain instead of wildard + // trim * and/or . (prefix) from the target to return the domain instead of wilcard target = strings.TrimPrefix(strings.Trim(target, "*"), ".") if !r.testAndSet(target) { return @@ -833,54 +833,42 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { return } for _, ip := range cidrIps { - results <- ip + results <- httpx.Target{Host: ip} } } else if r.options.ProbeAllIPS { URL, err := urlutil.Parse(target) if err != nil { - results <- target + results <- httpx.Target{Host: target} } ips, _, err := getDNSData(hp, URL.Host) if err != nil || len(ips) == 0 { - results <- target + results <- httpx.Target{Host: target} } for _, ip := range ips { - results <- strings.Join([]string{ip, target}, ",") + results <- httpx.Target{Host: target, CustomIP: ip} } + } else if idxComma := strings.Index(target, ","); idxComma > 0 { + results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]} } else { - results <- target + results <- httpx.Target{Host: target} } }() return results } -func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method, origInput string, scanopts *scanOptions) Result { +func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *scanOptions) Result { origProtocol := protocol if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS { protocol = httpx.HTTPS } retried := false retry: - var customHost, customIP string - if scanopts.ProbeAllIPS { - parts := strings.SplitN(domain, ",", 2) - if len(parts) == 2 { - customIP = parts[0] - domain = parts[1] - } + if scanopts.VHostInput && target.CustomHost == "" { + return Result{Input: origInput} } - if scanopts.VHostInput { - parts := strings.Split(domain, ",") - //nolint:gomnd // not a magic number - if len(parts) != 2 { - return Result{Input: origInput} - } - domain = parts[0] - customHost = parts[1] - } - URL, err := urlutil.Parse(domain) + URL, err := urlutil.Parse(target.Host) if err != nil { - return Result{URL: domain, Input: origInput, err: err} + return Result{URL: target.Host, Input: origInput, err: err} } // check if we have to skip the host:port as a result of a previous failure @@ -888,19 +876,19 @@ retry: if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { - return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} + return Result{URL: target.Host, err: errors.New("skipping as previously unresponsive")} } } // check if the combination host:port should be skipped if belonging to a cdn if r.skipCDNPort(URL.Host, URL.Port) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port) - return Result{URL: domain, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")} + return Result{URL: target.Host, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")} } URL.Scheme = protocol - if !strings.Contains(domain, URL.Port) { + if !strings.Contains(target.Host, URL.Port) { URL.Port = "" } @@ -915,9 +903,14 @@ retry: URL.RequestURI += scanopts.RequestURI } var req *retryablehttp.Request - if customIP != "" { - customHost = URL.Host - ctx := context.WithValue(context.Background(), "ip", customIP) //nolint + if target.CustomIP != "" { + var requestIP string + if iputil.IsIPv6(target.CustomIP) { + requestIP = fmt.Sprintf("[%s]", target.CustomIP) + } else { + requestIP = target.CustomIP + } + ctx := context.WithValue(context.Background(), "ip", requestIP) //nolint req, err = hp.NewRequestWithContext(ctx, method, URL.String()) } else { req, err = hp.NewRequest(method, URL.String()) @@ -926,8 +919,8 @@ retry: return Result{URL: URL.String(), Input: origInput, err: err} } - if customHost != "" { - req.Host = customHost + if target.CustomHost != "" { + req.Host = target.CustomHost } if !scanopts.LeaveDefaultPorts { @@ -1198,7 +1191,15 @@ retry: r.stats.IncrementCounter("requests", 1) } } - ip := hp.Dialer.GetDialedIP(URL.Host) + + var ip string + if target.CustomIP != "" { + ip = target.CustomIP + } else { + // hp.Dialer.GetDialedIP would return only the last dialed one + ip = hp.Dialer.GetDialedIP(URL.Host) + } + var asnResponse interface{ String() string } if r.options.Asn { lookupResult, err := ipisp.LookupIP(context.Background(), net.ParseIP(ip)) @@ -1222,10 +1223,7 @@ retry: builder.WriteRune(']') } } - // hp.Dialer.GetDialedIP would return only the last dialed one - if customIP != "" { - ip = customIP - } + if scanopts.OutputIP || scanopts.ProbeAllIPS { builder.WriteString(fmt.Sprintf(" [%s]", ip)) } From 60ca13fdccfc23423688bb1d583e56b6dd8bbcdd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 May 2022 15:43:53 +0530 Subject: [PATCH 191/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#635) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.42 to 0.0.43. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.42...v0.0.43) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 361c466..a4066ec 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.42 + github.com/projectdiscovery/wappalyzergo v0.0.43 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 8d67df5..82fb4b7 100644 --- a/go.sum +++ b/go.sum @@ -178,8 +178,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.42 h1:4DuJVpgesHSnUNVG+l62QIYEmcMXu3PLyhS1Wp75c30= -github.com/projectdiscovery/wappalyzergo v0.0.42/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.43 h1:Y+XBRfCxgAHsrP8fXAugLEELUGKe1jj+Gh5X2qKQvxY= +github.com/projectdiscovery/wappalyzergo v0.0.43/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From a924673e72f7378f22513a7e49a8d66358c4a358 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 May 2022 14:59:29 +0530 Subject: [PATCH 192/355] chore(deps): bump goreleaser/goreleaser-action from 2 to 3 (#634) Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 2 to 3. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/v2...v3) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index d709397..855356f 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -20,7 +20,7 @@ jobs: go-version: 1.18 - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v2 + uses: goreleaser/goreleaser-action@v3 with: args: "release --rm-dist" version: latest From c486967221bcfcb050c9f37337a5adeb81795202 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 May 2022 15:55:49 +0530 Subject: [PATCH 193/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#642) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.43 to 0.0.44. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.43...v0.0.44) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a4066ec..21a9f6a 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.43 + github.com/projectdiscovery/wappalyzergo v0.0.44 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 82fb4b7..2642341 100644 --- a/go.sum +++ b/go.sum @@ -178,8 +178,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.43 h1:Y+XBRfCxgAHsrP8fXAugLEELUGKe1jj+Gh5X2qKQvxY= -github.com/projectdiscovery/wappalyzergo v0.0.43/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.44 h1:L1CpNYHZF5RP/uNR83L3EkZGW13cP0WkVEgDk3vN1Sk= +github.com/projectdiscovery/wappalyzergo v0.0.44/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From fdf907688dc6a05e127473c6754f7adb698c4459 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 May 2022 15:56:01 +0530 Subject: [PATCH 194/355] chore(deps): bump alpine from 3.15.4 to 3.16.0 (#641) Bumps alpine from 3.15.4 to 3.16.0. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 233d5eb..991ec93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.18.2-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.15.4 +FROM alpine:3.16.0 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From ee655bbdcd68f37dc064c0e3b92aafada9d69878 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 31 May 2022 00:13:31 +0200 Subject: [PATCH 195/355] Adding Client TLS1.0 (#645) --- common/httpx/httpx.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 16e3360..0eb0a70 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -110,6 +110,7 @@ func New(options *Options) (*HTTPX, error) { MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, + MinVersion: tls.VersionTLS10, }, DisableKeepAlives: true, } @@ -134,6 +135,7 @@ func New(options *Options) (*HTTPX, error) { transport2 := &http2.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, + MinVersion: tls.VersionTLS10, }, AllowHTTP: true, } From b52da3aca6d637fef9cf9d4ac03d21257cb9061b Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 2 Jun 2022 12:28:44 -0500 Subject: [PATCH 196/355] changed warning to error if the directory does not have write permission (#650) --- runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index e7be198..610d2f1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1400,7 +1400,7 @@ retry: } writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644) if writeErr != nil { - gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) + gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } if scanopts.StoreChain && resp.HasChain() { domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") From 3743d2fefd08e9a09729a2ae06418195e60254b3 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Sat, 4 Jun 2022 12:31:29 -0500 Subject: [PATCH 197/355] updates from urlutil library (#646) * updates from urlutil library * urlutil dependency update * go mod tidy Co-authored-by: mzack --- go.mod | 5 +++-- go.sum | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 21a9f6a..31cb92a 100644 --- a/go.mod +++ b/go.mod @@ -27,10 +27,10 @@ require ( github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 github.com/projectdiscovery/mapcidr v0.0.9 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e - github.com/projectdiscovery/retryablehttp-go v1.0.2 + github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 - github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 + github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 github.com/projectdiscovery/wappalyzergo v0.0.44 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 @@ -53,6 +53,7 @@ require ( ) require ( + github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect diff --git a/go.sum b/go.sum index 2642341..f0553cf 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= @@ -166,8 +168,9 @@ github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/g github.com/projectdiscovery/retryabledns v1.0.13 h1:Ogfv0fl3Nszb+Nq2S2qQmE+PJDlStVxyLkmTotMohZY= github.com/projectdiscovery/retryabledns v1.0.13/go.mod h1:EeqHcAPp0g2GljT4qkxKSAE47Dj0ZrJQ46R9ct3Muhk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= -github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= +github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= +github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= @@ -176,8 +179,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4fvUw6b4sS4GoWbHr60mJo3dI//4mGt3BuLx8Sz9aNw= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= -github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= +github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= +github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= github.com/projectdiscovery/wappalyzergo v0.0.44 h1:L1CpNYHZF5RP/uNR83L3EkZGW13cP0WkVEgDk3vN1Sk= github.com/projectdiscovery/wappalyzergo v0.0.44/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= From 886c0ac73312cf31638e649f221018529a451c8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 17:02:52 +0530 Subject: [PATCH 198/355] chore(deps): bump golang from 1.18.2-alpine to 1.18.3-alpine (#654) Bumps golang from 1.18.2-alpine to 1.18.3-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 991ec93..b2e1ab5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18.2-alpine AS builder +FROM golang:1.18.3-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 103cdbea73153cfc3ab6c1a7abaa9f4a7c798674 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jun 2022 17:03:01 +0530 Subject: [PATCH 199/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#655) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.44 to 0.0.45. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.44...v0.0.45) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 31cb92a..447c00b 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.44 + github.com/projectdiscovery/wappalyzergo v0.0.45 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index f0553cf..df8187b 100644 --- a/go.sum +++ b/go.sum @@ -181,8 +181,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.44 h1:L1CpNYHZF5RP/uNR83L3EkZGW13cP0WkVEgDk3vN1Sk= -github.com/projectdiscovery/wappalyzergo v0.0.44/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.45 h1:b2OJ4/FrQg6YBDwuK+xMJj/3+h6u+uHuZYURGYPqw3I= +github.com/projectdiscovery/wappalyzergo v0.0.45/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From c1e9a1036c0592ace42d672d1c64e0d7f2f91c12 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jun 2022 10:09:47 +0000 Subject: [PATCH 200/355] chore(deps): bump github.com/projectdiscovery/mapcidr Bumps [github.com/projectdiscovery/mapcidr](https://github.com/projectdiscovery/mapcidr) from 0.0.9 to 1.0.0. - [Release notes](https://github.com/projectdiscovery/mapcidr/releases) - [Changelog](https://github.com/projectdiscovery/mapcidr/blob/master/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/mapcidr/compare/v0.0.9...v1.0.0) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/mapcidr dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 23 ++++++++++++----------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 447c00b..c27b256 100644 --- a/go.mod +++ b/go.mod @@ -18,18 +18,18 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc + github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 + github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 - github.com/projectdiscovery/mapcidr v0.0.9 + github.com/projectdiscovery/mapcidr v1.0.0 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea - github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 + github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 github.com/projectdiscovery/wappalyzergo v0.0.45 github.com/remeh/sizedwaitgroup v1.0.0 @@ -81,5 +81,5 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index df8187b..e598502 100644 --- a/go.sum +++ b/go.sum @@ -133,13 +133,12 @@ github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0 github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc h1:dbDgsj26PW06L3zMo7AT08IbEqMd2u8QQ1BvlfMAY2w= -github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= +github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483 h1:0as/e36wk1cnnqK3FojbdOx7fsm9yH6Gy5q9v6Sg/88= +github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483/go.mod h1:wjS/oBWbzlayJ/aTK0KW0oOHGO03G8oEYzuN6stI8Ho= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= -github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 h1:M9DJMe3zMnvHkt5McU1IpoTQbBYc2aB1nsOBLqWlQOs= -github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= +github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c h1:EU5CO4ZbtEp+xV3Cr308A759d0CREgwCnxqvxNMKzdk= +github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -156,8 +155,8 @@ github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kA github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v0.0.9 h1:PIa09fMHdghlmkUeTgHP9bwYnb3k2wXXM2f6LMj26zg= -github.com/projectdiscovery/mapcidr v0.0.9/go.mod h1:zgsrc+UXwcLcBopUNboiI4tpTICbfdTyJZiBi2tx+NI= +github.com/projectdiscovery/mapcidr v1.0.0 h1:4ctYAgkxaLeky3o3c+qMgHoUjytYy6DME2fjbgiB1Ag= +github.com/projectdiscovery/mapcidr v1.0.0/go.mod h1:5QkKrV6rNQQurCZI3nNedFsAOYp04mRDkC5yht+znYA= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= @@ -177,8 +176,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4fvUw6b4sS4GoWbHr60mJo3dI//4mGt3BuLx8Sz9aNw= -github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 h1:yo7hCL47BOHl8X/aMmPeRQwiqUrH6TZ2WjgqItaSPcc= +github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= github.com/projectdiscovery/wappalyzergo v0.0.45 h1:b2OJ4/FrQg6YBDwuK+xMJj/3+h6u+uHuZYURGYPqw3I= @@ -207,8 +206,9 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -345,5 +345,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 8a96d08aca531201eef97e1e1e0ce63595edb9e4 Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 12 Jun 2022 09:46:52 +0530 Subject: [PATCH 201/355] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 47b86df..6c09f11 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.1 + /_/ v1.2.2 ` // Version is the current version of httpx -const Version = `v1.2.1` +const Version = `v1.2.2` // showBanner is used to show the banner to the user func showBanner() { From 08a2be6e5f98b48a29b673af17ab5f929e4baaad Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 12 Jun 2022 09:50:36 +0530 Subject: [PATCH 202/355] readme update --- README.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 872ba2d..ae5ae2d 100644 --- a/README.md +++ b/README.md @@ -109,15 +109,15 @@ PROBES: -probe display probe status MATCHERS: - -mc, -match-code string match response with specified status code (-mc 200,302) - -ml, -match-length string match response with specified content length (-ml 100,102) - -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) - -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) - -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) - -ms, -match-string string match response with specified string (-ms admin) - -mr, -match-regex string match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (-mcdn azure,google) - -mrt, -match-response-time string match response with specified response time in seconds (-mrt '> 1') + -mc, -match-code string match response with specified status code (-mc 200,302) + -ml, -match-length string match response with specified content length (-ml 100,102) + -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) + -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) + -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) + -ms, -match-string string match response with specified string (-ms admin) + -mr, -match-regex string match response with specified regex (-mr admin) + -mcdn, -match-cdn string[] match host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb) + -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') EXTRACTOR: -er, -extract-regex string display response content for specified regex @@ -130,7 +130,7 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) -fs, -filter-string string filter response with specified string (-fs admin) -fe, -filter-regex string filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (-fcdn oracle,cloudflare) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') RATE-LIMIT: @@ -163,6 +163,7 @@ CONFIGURATIONS: -r, -resolvers string[] list of custom resolver (file or comma separated) -allow string[] allowed list of IP/CIDR's to process (file or comma separated) -deny string[] denied list of IP/CIDR's to process (file or comma separated) + -sni, -sni-name string Custom TLS SNI name -random-agent Enable Random User-Agent to use (default true) -H, -header string[] custom http headers to send with request -http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080) @@ -176,7 +177,7 @@ CONFIGURATIONS: -body string post body to include in http request -s, -stream stream mode - start elaborating input targets without sorting -sd, -skip-dedupe disable dedupe input items (only used with stream mode) - -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443) + -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443 DEBUG: -debug display request/response content in cli From 2becc11abc698922042d67c6248b35dac0afe5ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 10:09:16 +0000 Subject: [PATCH 203/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.45 to 0.0.46. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.45...v0.0.46) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 447c00b..61ab844 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.45 + github.com/projectdiscovery/wappalyzergo v0.0.46 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index df8187b..437108f 100644 --- a/go.sum +++ b/go.sum @@ -181,8 +181,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4f github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.45 h1:b2OJ4/FrQg6YBDwuK+xMJj/3+h6u+uHuZYURGYPqw3I= -github.com/projectdiscovery/wappalyzergo v0.0.45/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.46 h1:O+Ddxtx2q046P1Csu+ANWpOyn7+KD/h0gkRYVLm197I= +github.com/projectdiscovery/wappalyzergo v0.0.46/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From ff94a0baf5986fba0f37be90025b95181c49c4e4 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 13 Jun 2022 13:31:26 +0200 Subject: [PATCH 204/355] fixing go mod --- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index c27b256..57f27f2 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 - github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 + github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 github.com/projectdiscovery/mapcidr v1.0.0 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d @@ -65,7 +65,7 @@ require ( github.com/karrick/godirwalk v1.16.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect + github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.13 // indirect diff --git a/go.sum b/go.sum index e598502..ff43a41 100644 --- a/go.sum +++ b/go.sum @@ -119,8 +119,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= +github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d h1:1Q94movG4VMVo+qf781ttXXt2bxBOIGMenmuCAJDLfE= +github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= @@ -149,8 +150,9 @@ github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:Gzruq github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= -github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 h1:VZ9H51A7tI7R/9I5W5l960Nkq7eMJqGd3y1wsuwzdjE= github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU= +github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 h1:PSYGmqVDxBe9Pnbe14MIX7NXTHJqxMwBLzwDNBwSZYk= +github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619/go.mod h1:w5bXH/PHqM8/vF0bFN6fZMggm7IlPKK2dJdz5McY+Qw= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= From cd18edc689580a75934e59918a73b9ddba5f326d Mon Sep 17 00:00:00 2001 From: M09Ic Date: Mon, 13 Jun 2022 20:18:00 +0800 Subject: [PATCH 205/355] support multi -extract-regex flag and json output (#529) * Add hashes flag, to support multi body hash type. issue:489 and issue:488 * go mod update * Add hash type validate for -hash flag * support multi extract regexps * support extract regexps matches for json output * support extract regexp preset * extract result deduplicate * small changes * lint ignore Co-authored-by: sandeep Co-authored-by: mzack --- common/customextract/customextract.go | 9 +++ common/customextract/doc.go | 1 + runner/options.go | 8 ++- runner/runner.go | 98 +++++++++++++++++---------- 4 files changed, 77 insertions(+), 39 deletions(-) create mode 100644 common/customextract/customextract.go create mode 100644 common/customextract/doc.go diff --git a/common/customextract/customextract.go b/common/customextract/customextract.go new file mode 100644 index 0000000..d15afca --- /dev/null +++ b/common/customextract/customextract.go @@ -0,0 +1,9 @@ +package customextract + +import "regexp" + +var ExtractPresets = map[string]*regexp.Regexp{ + "url": regexp.MustCompile("^(http(s)?:\\/\\/)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+(:[0-9]{1,5})?[-a-zA-Z0-9()@:%_\\\\\\+\\.~#?&//=]*$"), //nolint + "ip": regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}"), //nolint + "mail": regexp.MustCompile("^([A-Za-z0-9_\\-\\.\u4e00-\u9fa5])+\\@([A-Za-z0-9_\\-\\.])+\\.([A-Za-z]{2,8})$"), //nolint +} diff --git a/common/customextract/doc.go b/common/customextract/doc.go new file mode 100644 index 0000000..20f24a8 --- /dev/null +++ b/common/customextract/doc.go @@ -0,0 +1 @@ +package customextract diff --git a/runner/options.go b/runner/options.go index 25407e3..43132b2 100644 --- a/runner/options.go +++ b/runner/options.go @@ -68,7 +68,7 @@ type scanOptions struct { MaxResponseBodySizeToSave int MaxResponseBodySizeToRead int OutputExtractRegex string - extractRegex *regexp.Regexp + extractRegexps map[string]*regexp.Regexp ExcludeCDN bool HostMaxErrors int ProbeAllIPS bool @@ -205,7 +205,8 @@ type Options struct { Allow customlist.CustomList MaxResponseBodySizeToSave int MaxResponseBodySizeToRead int - OutputExtractRegex string + OutputExtractRegexs goflags.StringSlice + OutputExtractPresets goflags.StringSlice RateLimit int RateLimitMinute int Probe bool @@ -289,7 +290,8 @@ func ParseOptions() *Options { ) createGroup(flagSet, "extractor", "Extractor", - flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "display response content for specified regex"), + flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "Display response content with matched regex"), + flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, "Display response content with matched preset regex"), ) createGroup(flagSet, "filters", "Filters", diff --git a/runner/runner.go b/runner/runner.go index 610d2f1..d046aa9 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -22,6 +22,9 @@ import ( "strings" "time" + "github.com/projectdiscovery/httpx/common/customextract" + "github.com/projectdiscovery/sliceutil" + "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" "github.com/logrusorgru/aurora" @@ -212,9 +215,25 @@ func New(options *Options) (*Runner, error) { scanopts.StoreChain = options.StoreChain scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave scanopts.MaxResponseBodySizeToRead = options.MaxResponseBodySizeToRead - if options.OutputExtractRegex != "" { - if scanopts.extractRegex, err = regexp.Compile(options.OutputExtractRegex); err != nil { - return nil, err + scanopts.extractRegexps = make(map[string]*regexp.Regexp) + + if options.OutputExtractRegexs != nil { + for _, regex := range options.OutputExtractRegexs { + if compiledRegex, err := regexp.Compile(regex); err != nil { + return nil, err + } else { + scanopts.extractRegexps[regex] = compiledRegex + } + } + } + + if options.OutputExtractPresets != nil { + for _, regexName := range options.OutputExtractPresets { + if regex, ok := customextract.ExtractPresets[regexName]; ok { + scanopts.extractRegexps[regexName] = regex + } else { + gologger.Warning().Msgf("Could not find preset: %s\n", regexName) + } } } @@ -1269,10 +1288,15 @@ retry: } // extract regex - if scanopts.extractRegex != nil { - matches := scanopts.extractRegex.FindAllString(string(resp.Data), -1) - if len(matches) > 0 { - builder.WriteString(" [" + strings.Join(matches, ",") + "]") + var extractResult = map[string][]string{} + if scanopts.extractRegexps != nil { + for regex, compiledRegex := range scanopts.extractRegexps { + matches := compiledRegex.FindAllString(string(resp.Data), -1) + if len(matches) > 0 { + matches = sliceutil.Dedupe(matches) + builder.WriteString(" [" + strings.Join(matches, ",") + "]") + extractResult[regex] = matches + } } } @@ -1475,6 +1499,7 @@ retry: FinalURL: finalURL, FavIconMMH3: faviconMMH3, Hashes: hashesMap, + Extracts: extractResult, Jarm: jarmhash, Lines: resp.Lines, Words: resp.Words, @@ -1517,35 +1542,36 @@ type Result struct { Location string `json:"location,omitempty" csv:"location"` Title string `json:"title,omitempty" csv:"title"` str string - Scheme string `json:"scheme,omitempty" csv:"scheme"` - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"response-body,omitempty" csv:"response-body"` - ContentType string `json:"content-type,omitempty" csv:"content-type"` - Method string `json:"method,omitempty" csv:"method"` - Host string `json:"host,omitempty" csv:"host"` - Path string `json:"path,omitempty" csv:"path"` - FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` - FinalURL string `json:"final-url,omitempty" csv:"final-url"` - ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` - Request string `json:"request,omitempty" csv:"request"` - ResponseTime string `json:"response-time,omitempty" csv:"response-time"` - Jarm string `json:"jarm,omitempty" csv:"jarm"` - ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` - A []string `json:"a,omitempty" csv:"a"` - CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` - Technologies []string `json:"technologies,omitempty" csv:"technologies"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - Words int `json:"words" csv:"words"` - Lines int `json:"lines" csv:"lines"` - StatusCode int `json:"status-code,omitempty" csv:"status-code"` - ContentLength int `json:"content-length,omitempty" csv:"content-length"` - Failed bool `json:"failed" csv:"failed"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` + Scheme string `json:"scheme,omitempty" csv:"scheme"` + Error string `json:"error,omitempty" csv:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver"` + ResponseBody string `json:"response-body,omitempty" csv:"response-body"` + ContentType string `json:"content-type,omitempty" csv:"content-type"` + Method string `json:"method,omitempty" csv:"method"` + Host string `json:"host,omitempty" csv:"host"` + Path string `json:"path,omitempty" csv:"path"` + FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` + FinalURL string `json:"final-url,omitempty" csv:"final-url"` + ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` + Request string `json:"request,omitempty" csv:"request"` + ResponseTime string `json:"response-time,omitempty" csv:"response-time"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` + ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` + A []string `json:"a,omitempty" csv:"a"` + CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` + Technologies []string `json:"technologies,omitempty" csv:"technologies"` + Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` + Words int `json:"words" csv:"words"` + Lines int `json:"lines" csv:"lines"` + StatusCode int `json:"status-code,omitempty" csv:"status-code"` + ContentLength int `json:"content-length,omitempty" csv:"content-length"` + Failed bool `json:"failed" csv:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` } // JSON the result From 2c0802dc7b5f5fe1bcee94298a046d0419a9619f Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 13 Jun 2022 14:18:28 +0200 Subject: [PATCH 206/355] Fixing wrong app name (#663) --- cmd/functional-test/run.sh | 6 +++--- sonar-project.properties | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/functional-test/run.sh b/cmd/functional-test/run.sh index 2d4ddb8..cd1640b 100755 --- a/cmd/functional-test/run.sh +++ b/cmd/functional-test/run.sh @@ -11,13 +11,13 @@ echo "::group::Building functional-test binary" go build -o functional-test$extension echo "::endgroup::" -echo "::group::Building dnsx binary from current branch" +echo "::group::Building httpx binary from current branch" go build -o httpx_dev$extension ../httpx echo "::endgroup::" -echo "::group::Building latest release of dnsx" +echo "::group::Building latest release of httpx" go build -o httpx$extension -v github.com/projectdiscovery/httpx/cmd/httpx echo "::endgroup::" -echo 'Starting dnsx functional test' +echo 'Starting httpx functional test' ./functional-test$extension -main ./httpx$extension -dev ./httpx_dev$extension -testcases testcases.txt diff --git a/sonar-project.properties b/sonar-project.properties index 1b99918..0a9a5a5 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -2,7 +2,7 @@ sonar.projectKey=projectdiscovery_httpx sonar.organization=projectdiscovery # This is the name and version displayed in the SonarCloud UI. -#sonar.projectName=dnsx +#sonar.projectName=httpx #sonar.projectVersion=1.0 # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. From 14162b4576af3d9a04a6635434282464d388bdc4 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 16 Jun 2022 10:46:15 +0200 Subject: [PATCH 207/355] Add self diagnostic functionality (#665) --- go.mod | 12 ++++---- go.sum | 20 ++++++++++---- runner/healthcheck.go | 64 +++++++++++++++++++++++++++++++++++++++++++ runner/options.go | 36 ++++++++++++------------ 4 files changed, 102 insertions(+), 30 deletions(-) create mode 100644 runner/healthcheck.go diff --git a/go.mod b/go.mod index f8f9a70..b26e959 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,9 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483 + github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c + github.com/projectdiscovery/goflags v0.0.8-0.20220610073650-5d31a8c159e3 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea - github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 + github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 github.com/projectdiscovery/wappalyzergo v0.0.46 github.com/remeh/sizedwaitgroup v1.0.0 @@ -38,8 +38,8 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/net v0.0.0-20220607020251-c690dde0001d + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 ) @@ -62,13 +62,13 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/karrick/godirwalk v1.16.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.13 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 // indirect diff --git a/go.sum b/go.sum index 967f2da..58e8848 100644 --- a/go.sum +++ b/go.sum @@ -68,7 +68,6 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -134,12 +133,16 @@ github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0 github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483 h1:0as/e36wk1cnnqK3FojbdOx7fsm9yH6Gy5q9v6Sg/88= github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483/go.mod h1:wjS/oBWbzlayJ/aTK0KW0oOHGO03G8oEYzuN6stI8Ho= +github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c h1:/+xJK8e/Gj/zAmudWh5l2SzGJB+CkwYnraelCkBe7Aw= +github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c h1:EU5CO4ZbtEp+xV3Cr308A759d0CREgwCnxqvxNMKzdk= github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= +github.com/projectdiscovery/goflags v0.0.8-0.20220610073650-5d31a8c159e3 h1:2cUSicd6wkd7A3P+tjjAYiPmD99FtWp4k2curJkcrJ0= +github.com/projectdiscovery/goflags v0.0.8-0.20220610073650-5d31a8c159e3/go.mod h1:Np0EwqmopHDDb1cY4/NEBaC4fornMIWc6dr5yL91kzM= +github.com/projectdiscovery/goflags v0.0.8 h1:IhTmnEGSKtBHfD23tSwpnzai5CRYfLKVOWlVq9ngYvY= +github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -178,8 +181,9 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833 h1:yo7hCL47BOHl8X/aMmPeRQwiqUrH6TZ2WjgqItaSPcc= github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:EbP+rVR7NPsFKfwhzshmXP2GMjqBpaMZurJzQ5PYvbY= +github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= github.com/projectdiscovery/wappalyzergo v0.0.46 h1:O+Ddxtx2q046P1Csu+ANWpOyn7+KD/h0gkRYVLm197I= @@ -191,6 +195,8 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -266,8 +272,9 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -299,8 +306,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/runner/healthcheck.go b/runner/healthcheck.go new file mode 100644 index 0000000..c0192f9 --- /dev/null +++ b/runner/healthcheck.go @@ -0,0 +1,64 @@ +package runner + +import ( + "fmt" + "net" + "runtime" + "strings" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/goflags" +) + +func DoHealthCheck(options *Options) string { + // RW permissions on config file + cfgFilePath, _ := goflags.GetConfigFilePath() + var test strings.Builder + test.WriteString(fmt.Sprintf("Version: %s\n", Version)) + test.WriteString(fmt.Sprintf("Operative System: %s\n", runtime.GOOS)) + test.WriteString(fmt.Sprintf("Architecture: %s\n", runtime.GOARCH)) + test.WriteString(fmt.Sprintf("Go Version: %s\n", runtime.Version())) + test.WriteString(fmt.Sprintf("Compiler: %s\n", runtime.Compiler)) + + var testResult string + ok, err := fileutil.IsReadable(cfgFilePath) + if ok { + testResult = "Ok" + } else { + testResult = "Ko" + } + if err != nil { + testResult += fmt.Sprintf(" (%s)", err) + } + test.WriteString(fmt.Sprintf("Config file \"%s\" Read => %s\n", cfgFilePath, testResult)) + ok, err = fileutil.IsWriteable(cfgFilePath) + if ok { + testResult = "Ok" + } else { + testResult = "Ko" + } + if err != nil { + testResult += fmt.Sprintf(" (%s)", err) + } + test.WriteString(fmt.Sprintf("Config file \"%s\" Write => %s\n", cfgFilePath, testResult)) + c4, err := net.Dial("tcp4", "scanme.sh:80") + if err == nil && c4 != nil { + c4.Close() + } + testResult = "Ok" + if err != nil { + testResult = fmt.Sprintf("Ko (%s)", err) + } + test.WriteString(fmt.Sprintf("IPv4 connectivity to scanme.sh:80 => %s\n", testResult)) + c6, err := net.Dial("tcp6", "scanme.sh:80") + if err == nil && c6 != nil { + c6.Close() + } + testResult = "Ok" + if err != nil { + testResult = fmt.Sprintf("Ko (%s)", err) + } + test.WriteString(fmt.Sprintf("IPv6 connectivity to scanme.sh:80 => %s\n", testResult)) + + return test.String() +} diff --git a/runner/options.go b/runner/options.go index 43132b2..5b8e9a4 100644 --- a/runner/options.go +++ b/runner/options.go @@ -240,6 +240,7 @@ type Options struct { SniName string OutputMatchResponseTime string OutputFilterResponseTime string + HealthCheck bool } // ParseOptions parses the command line options for application @@ -249,12 +250,12 @@ func ParseOptions() *Options { flagSet := goflags.NewFlagSet() flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library.`) - createGroup(flagSet, "input", "Input", + flagSet.CreateGroup("input", "Input", flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"), flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"), ) - createGroup(flagSet, "Probes", "Probes", + flagSet.CreateGroup("Probes", "Probes", flagSet.BoolVarP(&options.StatusCode, "status-code", "sc", false, "display response status-code"), flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "display response content-length"), flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "display response content-type"), @@ -277,7 +278,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"), ) - createGroup(flagSet, "matchers", "Matchers", + flagSet.CreateGroup("matchers", "Matchers", flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "match response with specified status code (-mc 200,302)"), flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"), flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"), @@ -289,12 +290,12 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"), ) - createGroup(flagSet, "extractor", "Extractor", + flagSet.CreateGroup("extractor", "Extractor", flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "Display response content with matched regex"), flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, "Display response content with matched preset regex"), ) - createGroup(flagSet, "filters", "Filters", + flagSet.CreateGroup("filters", "Filters", flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "filter response with specified status code (-fc 403,401)"), flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"), flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"), @@ -306,13 +307,13 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), ) - createGroup(flagSet, "rate-limit", "Rate-Limit", + flagSet.CreateGroup("rate-limit", "Rate-Limit", flagSet.IntVarP(&options.Threads, "threads", "t", 50, "number of threads to use"), flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 150, "maximum requests to send per second"), flagSet.IntVarP(&options.RateLimitMinute, "rate-limit-minute", "rlm", 0, "maximum number of requests to send per minute"), ) - createGroup(flagSet, "Misc", "Miscellaneous", + flagSet.CreateGroup("Misc", "Miscellaneous", flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "probe all the ips associated with same host"), flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg 1,2-10,11)"), flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"), @@ -324,7 +325,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.VHost, "vhost", false, "probe and display server supporting VHOST"), ) - createGroup(flagSet, "output", "Output", + flagSet.CreateGroup("output", "Output", flagSet.StringVarP(&options.Output, "output", "o", "", "file to write output results"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"), @@ -335,7 +336,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"), ) - createGroup(flagSet, "configs", "Configurations", + flagSet.CreateGroup("configs", "Configurations", flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "list of custom resolver (file or comma separated)"), flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), @@ -356,7 +357,8 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https//host:443"), ) - createGroup(flagSet, "debug", "Debug", + flagSet.CreateGroup("debug", "Debug", + flagSet.BoolVarP(&options.HealthCheck, "hc", "health-check", false, "run diagnostic check up"), flagSet.BoolVar(&options.Debug, "debug", false, "display request/response content in cli"), flagSet.BoolVar(&options.DebugRequests, "debug-req", false, "display request content in cli"), flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "display response content in cli"), @@ -368,7 +370,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable colors in cli output"), ) - createGroup(flagSet, "Optimizations", "Optimizations", + flagSet.CreateGroup("Optimizations", "Optimizations", flagSet.BoolVarP(&options.NoFallback, "no-fallback", "nf", false, "display both probed protocol (HTTPS and HTTP)"), flagSet.BoolVarP(&options.NoFallbackScheme, "no-fallback-scheme", "nfs", false, "probe with protocol scheme specified in input "), flagSet.IntVarP(&options.HostMaxErrors, "max-host-error", "maxhr", 30, "max error count per host before skipping remaining path/s"), @@ -381,6 +383,11 @@ func ParseOptions() *Options { _ = flagSet.Parse() + if options.HealthCheck { + gologger.Print().Msgf("%s\n", DoHealthCheck(options)) + os.Exit(0) + } + if options.StatsInterval != 0 { options.ShowStatistics = true } @@ -538,10 +545,3 @@ func (options *Options) ShouldLoadResume() bool { func (options *Options) ShouldSaveResume() bool { return true } - -func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { - flagSet.SetGroup(groupName, description) - for _, currentFlag := range flags { - currentFlag.Group(groupName) - } -} From 310294649f12f6ac3e4f1d4135d0e2de43902063 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 27 Jun 2022 10:06:42 +0200 Subject: [PATCH 208/355] Improving HTML decoding (#673) --- common/httpx/encodings.go | 37 +++++++++++++++++++++++++++++++ common/httpx/httpx.go | 5 +++++ common/httpx/title.go | 46 ++------------------------------------- go.sum | 2 -- 4 files changed, 44 insertions(+), 46 deletions(-) diff --git a/common/httpx/encodings.go b/common/httpx/encodings.go index f9396a2..752e2e8 100644 --- a/common/httpx/encodings.go +++ b/common/httpx/encodings.go @@ -3,7 +3,10 @@ package httpx import ( "bytes" "io/ioutil" + "net/http" + "strings" + "github.com/projectdiscovery/stringsutil" "golang.org/x/text/encoding/korean" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/encoding/traditionalchinese" @@ -49,3 +52,37 @@ func DecodeKorean(s []byte) ([]byte, error) { koreanDecoder := korean.EUCKR.NewDecoder() return koreanDecoder.Bytes(s) } + +// ExtractTitle from a response +func DecodeData(data []byte, headers http.Header) ([]byte, error) { + // Non UTF-8 + if contentTypes, ok := headers["Content-Type"]; ok { + contentType := strings.ToLower(strings.Join(contentTypes, ";")) + + switch { + case stringsutil.ContainsAny(contentType, "charset=gb2312", "charset=gbk"): + return Decodegbk([]byte(data)) + case stringsutil.ContainsAny(contentType, "euc-kr"): + return DecodeKorean(data) + } + + // Content-Type from head tag + var match = reContentType.FindSubmatch(data) + var mcontentType = "" + if len(match) != 0 { + for i, v := range match { + if string(v) != "" && i != 0 { + mcontentType = string(v) + } + } + mcontentType = strings.ToLower(mcontentType) + } + switch { + case stringsutil.ContainsAny(mcontentType, "gb2312", "gbk"): + return Decodegbk(data) + } + } + + // return as is + return data, nil +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 0eb0a70..4438e6f 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -213,6 +213,11 @@ get_response: return nil, closeErr } + respbody, err = DecodeData(respbody, httpresp.Header) + if err != nil && !shouldIgnoreBodyErrors { + return nil, closeErr + } + respbodystr := string(respbody) // check if we need to strip html diff --git a/common/httpx/title.go b/common/httpx/title.go index 88a2b34..7ad7cb1 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -35,51 +35,9 @@ func ExtractTitle(r *Response) (title string) { // remove unwanted chars title = strings.TrimSpace(strings.Trim(title, cutset)) - title = strings.ReplaceAll(title, "\n", "") - title = strings.ReplaceAll(title, "\r", "") + title = stringsutil.ReplaceAny(title, "\n", "\r") - // Non UTF-8 - if contentTypes, ok := r.Headers["Content-Type"]; ok { - contentType := strings.ToLower(strings.Join(contentTypes, ";")) - - switch { - case stringsutil.ContainsAny(contentType, "charset=gb2312", "charset=gbk"): - titleUtf8, err := Decodegbk([]byte(title)) - if err != nil { - return - } - - return string(titleUtf8) - case stringsutil.ContainsAny(contentType, "euc-kr"): - titleUtf8, err := DecodeKorean([]byte(title)) - if err != nil { - return - } - return string(titleUtf8) - } - - // Content-Type from head tag - var match = reContentType.FindSubmatch(r.Data) - var mcontentType = "" - if len(match) != 0 { - for i, v := range match { - if string(v) != "" && i != 0 { - mcontentType = string(v) - } - } - mcontentType = strings.ToLower(mcontentType) - } - switch { - case stringsutil.ContainsAny(mcontentType, "gb2312", "gbk"): - titleUtf8, err := Decodegbk([]byte(title)) - if err != nil { - return - } - return string(titleUtf8) - } - } - - return //nolint + return title } func getTitleWithDom(r *Response) (*html.Node, error) { diff --git a/go.sum b/go.sum index 58e8848..0b4dbda 100644 --- a/go.sum +++ b/go.sum @@ -141,8 +141,6 @@ github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= github.com/projectdiscovery/goflags v0.0.8-0.20220610073650-5d31a8c159e3 h1:2cUSicd6wkd7A3P+tjjAYiPmD99FtWp4k2curJkcrJ0= github.com/projectdiscovery/goflags v0.0.8-0.20220610073650-5d31a8c159e3/go.mod h1:Np0EwqmopHDDb1cY4/NEBaC4fornMIWc6dr5yL91kzM= -github.com/projectdiscovery/goflags v0.0.8 h1:IhTmnEGSKtBHfD23tSwpnzai5CRYfLKVOWlVq9ngYvY= -github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= From d2aea6d9f200f59b333961d0f439edc0355991a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jun 2022 13:37:24 +0530 Subject: [PATCH 209/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#674) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.46 to 0.0.48. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.46...v0.0.48) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index b26e959..5e18255 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.46 + github.com/projectdiscovery/wappalyzergo v0.0.48 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 0b4dbda..5ac01f2 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.46 h1:O+Ddxtx2q046P1Csu+ANWpOyn7+KD/h0gkRYVLm197I= -github.com/projectdiscovery/wappalyzergo v0.0.46/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/wappalyzergo v0.0.48 h1:+5w6LzAdqYq3gJs4c5BkPYzOOqC7zYvA40ZoNLg/b+g= +github.com/projectdiscovery/wappalyzergo v0.0.48/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -206,6 +206,7 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -213,8 +214,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= +github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= From 32930d2cc65e1b56612520876d99beba80bd599e Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 27 Jun 2022 10:08:11 +0200 Subject: [PATCH 210/355] Using scanme.sh in functional tests + Fixing gosec action error (#667) * Using scanme.sh in functional tests * using gosec action --- .github/workflows/sonarcloud.yml | 6 ++--- cmd/functional-test/test-data/request.txt | 2 +- cmd/functional-test/testcases.txt | 32 +++++++++++------------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index e0df696..20e7e48 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -27,9 +27,9 @@ jobs: go test -coverprofile=./cov.out ./... - name: Run Gosec Security Scanner - run: | - go install github.com/securego/gosec/cmd/gosec@latest - gosec -no-fail -fmt=sonarqube -out report.json ./... + uses: securego/gosec@master + with: + args: '-no-fail -fmt=sonarqube -out report.json ./...' - name: SonarCloud Scan uses: SonarSource/sonarcloud-github-action@master diff --git a/cmd/functional-test/test-data/request.txt b/cmd/functional-test/test-data/request.txt index 284d3f2..655c9e2 100644 --- a/cmd/functional-test/test-data/request.txt +++ b/cmd/functional-test/test-data/request.txt @@ -1 +1 @@ -https://www.example.com +https://scanme.sh diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index e95bdc3..0138562 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -1,16 +1,16 @@ -www.example.com {{binary}} -silent -www.example.com {{binary}} -silent -l test-data/request.txt -www.example.com {{binary}} -silent -request test-data/raw-request.txt -www.example.com {{binary}} -silent -title -www.example.com {{binary}} -silent -sc -www.example.com {{binary}} -silent -td -www.example.com {{binary}} -silent -probe -www.example.com {{binary}} -silent -no-fallback -www.example.com {{binary}} -silent -cl -www.example.com {{binary}} -silent -server -www.example.com {{binary}} -silent -ip -www.example.com {{binary}} -silent -tls-grab -www.example.com {{binary}} -silent -unsafe -www.example.com {{binary}} -silent -x all -www.example.com {{binary}} -silent -body 'a=b' -www.example.com {{binary}} -silent -exclude-cdn +scanme.sh {{binary}} -silent +scanme.sh {{binary}} -silent -l test-data/request.txt +scanme.sh {{binary}} -silent -request test-data/raw-request.txt +scanme.sh {{binary}} -silent -title +scanme.sh {{binary}} -silent -sc +scanme.sh {{binary}} -silent -td +scanme.sh {{binary}} -silent -probe +scanme.sh {{binary}} -silent -no-fallback +scanme.sh {{binary}} -silent -cl +scanme.sh {{binary}} -silent -server +scanme.sh {{binary}} -silent -ip +scanme.sh {{binary}} -silent -tls-grab +scanme.sh {{binary}} -silent -unsafe +scanme.sh {{binary}} -silent -x all +scanme.sh {{binary}} -silent -body 'a=b' +scanme.sh {{binary}} -silent -exclude-cdn From e4e2aab40211f03062a87ff67609b2340c283c93 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 28 Jun 2022 12:49:46 +0200 Subject: [PATCH 211/355] Adding httpx as a library minimal example (#670) --- README.md | 39 ++++++++++++++++++++++ cmd/integration-test/integration-test.go | 3 +- cmd/integration-test/library.go | 41 ++++++++++++++++++++++++ runner/options.go | 39 +++++++++++++--------- 4 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 cmd/integration-test/library.go diff --git a/README.md b/README.md index ae5ae2d..30297a7 100644 --- a/README.md +++ b/README.md @@ -451,6 +451,45 @@ https://docs.hackerone.com https://support.hackerone.com ``` +### Using httpx as a library +`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to close at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it: + +```go +package main + +import ( + "log" + "os" + + "github.com/projectdiscovery/httpx/runner" +) + +func main() { + inputFile := "test.txt" + err := os.WriteFile(inputFile, []byte("scanme.sh"), 0644) + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(inputFile) + + options := runner.Options{ + Methods: "GET", + InputFile: inputFile, + } + if err := options.ValidateOptions(); err != nil { + log.Fatal(err) + } + + httpxRunner, err := runner.New(&options) + if err != nil { + log.Fatal() + } + defer httpxRunner.Close() + + httpxRunner.RunEnumeration() +} +``` + # 📋 Notes diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index 0b5a57e..d5115e0 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -22,7 +22,8 @@ func main() { failed := aurora.Red("[✘]").String() tests := map[string]map[string]testutils.TestCase{ - "http": httpTestcases, + "http": httpTestcases, + "library": libraryTestcases, } for proto, tests := range tests { if protocol == "" || protocol == proto { diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go new file mode 100644 index 0000000..f20e4bf --- /dev/null +++ b/cmd/integration-test/library.go @@ -0,0 +1,41 @@ +package main + +import ( + "os" + + "github.com/projectdiscovery/httpx/internal/testutils" + "github.com/projectdiscovery/httpx/runner" +) + +var libraryTestcases = map[string]testutils.TestCase{ + "Httpx as library": &httpxLibrary{}, +} + +type httpxLibrary struct { +} + +func (h *httpxLibrary) Execute() error { + testFile := "test.txt" + err := os.WriteFile(testFile, []byte("scanme.sh"), 0644) + if err != nil { + return err + } + defer os.RemoveAll(testFile) + + options := runner.Options{ + Methods: "GET", + InputFile: testFile, + } + if err := options.ValidateOptions(); err != nil { + return err + } + + httpxRunner, err := runner.New(&options) + if err != nil { + return err + } + defer httpxRunner.Close() + + httpxRunner.RunEnumeration() + return nil +} diff --git a/runner/options.go b/runner/options.go index 5b8e9a4..a47db04 100644 --- a/runner/options.go +++ b/runner/options.go @@ -7,6 +7,8 @@ import ( "regexp" "strings" + "github.com/pkg/errors" + "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" @@ -407,59 +409,61 @@ func ParseOptions() *Options { os.Exit(0) } - options.validateOptions() + if err := options.ValidateOptions(); err != nil { + gologger.Fatal().Msgf("%s\n", err) + } return options } -func (options *Options) validateOptions() { +func (options *Options) ValidateOptions() error { if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { - gologger.Fatal().Msgf("File %s does not exist.\n", options.InputFile) + return fmt.Errorf("File %s does not exist.", options.InputFile) } if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) { - gologger.Fatal().Msgf("File %s does not exist.\n", options.InputRawRequest) + return fmt.Errorf("File %s does not exist.", options.InputRawRequest) } multiOutput := options.CSVOutput && options.JSONOutput if multiOutput { - gologger.Fatal().Msg("Results can only be displayed in one format: 'JSON' or 'CSV'\n") + return fmt.Errorf("Results can only be displayed in one format: 'JSON' or 'CSV'") } var err error if options.matchStatusCode, err = stringz.StringToSliceInt(options.OutputMatchStatusCode); err != nil { - gologger.Fatal().Msgf("Invalid value for match status code option: %s\n", err) + return errors.Wrap(err, "Invalid value for match status code option") } if options.matchContentLength, err = stringz.StringToSliceInt(options.OutputMatchContentLength); err != nil { - gologger.Fatal().Msgf("Invalid value for match content length option: %s\n", err) + return errors.Wrap(err, "Invalid value for match content length option") } if options.filterStatusCode, err = stringz.StringToSliceInt(options.OutputFilterStatusCode); err != nil { - gologger.Fatal().Msgf("Invalid value for filter status code option: %s\n", err) + return errors.Wrap(err, "Invalid value for filter status code option") } if options.filterContentLength, err = stringz.StringToSliceInt(options.OutputFilterContentLength); err != nil { - gologger.Fatal().Msgf("Invalid value for filter content length option: %s\n", err) + return errors.Wrap(err, "Invalid value for filter content length option") } if options.OutputFilterRegex != "" { if options.filterRegex, err = regexp.Compile(options.OutputFilterRegex); err != nil { - gologger.Fatal().Msgf("Invalid value for regex filter option: %s\n", err) + return errors.Wrap(err, "Invalid value for regex filter option") } } if options.OutputMatchRegex != "" { if options.matchRegex, err = regexp.Compile(options.OutputMatchRegex); err != nil { - gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err) + return errors.Wrap(err, "Invalid value for match regex option") } } if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil { - gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err) + return errors.Wrap(err, "Invalid value for match lines count option") } if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil { - gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err) + return errors.Wrap(err, "Invalid value for match words count option") } if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil { - gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err) + return errors.Wrap(err, "Invalid value for filter lines count option") } if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil { - gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err) + return errors.Wrap(err, "Invalid value for filter words count option") } var resolvers []string @@ -467,7 +471,7 @@ func (options *Options) validateOptions() { if fileutil.FileExists(resolver) { chFile, err := fileutil.ReadFile(resolver) if err != nil { - gologger.Fatal().Msgf("Couldn't process resolver file \"%s\": %s\n", resolver, err) + return errors.Wrapf(err, "Couldn't process resolver file \"%s\"", resolver) } for line := range chFile { resolvers = append(resolvers, line) @@ -476,6 +480,7 @@ func (options *Options) validateOptions() { resolvers = append(resolvers, resolver) } } + options.Resolvers = resolvers if len(options.Resolvers) > 0 { gologger.Debug().Msgf("Using resolvers: %s\n", strings.Join(options.Resolvers, ",")) @@ -505,6 +510,8 @@ func (options *Options) validateOptions() { if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 { options.OutputCDN = true } + + return nil } // configureOutput configures the output on the screen From ba6220d296fd0639a03255c6bcc359ceaca68162 Mon Sep 17 00:00:00 2001 From: Dothelm <49758889+st3rv04ka@users.noreply.github.com> Date: Tue, 28 Jun 2022 15:04:26 +0400 Subject: [PATCH 212/355] Exposing regex matches to json output (#668) * Update runner.go Fix regex json output. * small refactor Co-authored-by: mzack --- runner/runner.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 610d2f1..e640b59 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -31,6 +31,7 @@ import ( "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/sliceutil" "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" @@ -1268,11 +1269,13 @@ retry: } } + var extractRegex []string // extract regex if scanopts.extractRegex != nil { - matches := scanopts.extractRegex.FindAllString(string(resp.Data), -1) - if len(matches) > 0 { - builder.WriteString(" [" + strings.Join(matches, ",") + "]") + extractRegex = scanopts.extractRegex.FindAllString(string(resp.Data), -1) + extractRegex = sliceutil.Dedupe(extractRegex) + if len(extractRegex) > 0 { + builder.WriteString(" [" + strings.Join(extractRegex, ",") + "]") } } @@ -1479,6 +1482,7 @@ retry: Lines: resp.Lines, Words: resp.Words, ASN: asnResponse, + ExtractRegex: extractRegex, } } @@ -1509,6 +1513,7 @@ type Result struct { CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty" csv:"tls-grab"` Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` + ExtractRegex []string `json:"extract-regex,omitempty" csv:"regex"` CDNName string `json:"cdn-name,omitempty" csv:"cdn-name"` Port string `json:"port,omitempty" csv:"port"` raw string From dfab811ff997ccfb23101919b9d912b293d48c31 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 5 Jul 2022 20:07:08 +0200 Subject: [PATCH 213/355] removing redundant regex (#676) --- common/hashes/jarmhash.go | 11 +++-------- common/regexhelper/regex.go | 7 ------- 2 files changed, 3 insertions(+), 15 deletions(-) delete mode 100644 common/regexhelper/regex.go diff --git a/common/hashes/jarmhash.go b/common/hashes/jarmhash.go index e1d48d1..56c3936 100644 --- a/common/hashes/jarmhash.go +++ b/common/hashes/jarmhash.go @@ -9,7 +9,6 @@ import ( "time" "github.com/RumbleDiscovery/jarm-go" - "github.com/projectdiscovery/httpx/common/regexhelper" "golang.org/x/net/proxy" ) @@ -20,8 +19,8 @@ var DefualtBackoff = func(r, m int) time.Duration { } type target struct { - Host string - Port int + Host string + Port int Retries int Backoff func(r, m int) time.Duration } @@ -85,9 +84,5 @@ func Jarm(host string, duration int) string { if t.Port == 0 { t.Port = defaultPort } - hash := fingerprint(t, duration) - if regexhelper.JarmHashRegex.MatchString(hash) { - return "" - } - return hash + return fingerprint(t, duration) } diff --git a/common/regexhelper/regex.go b/common/regexhelper/regex.go deleted file mode 100644 index 7bdd81a..0000000 --- a/common/regexhelper/regex.go +++ /dev/null @@ -1,7 +0,0 @@ -package regexhelper - -import "regexp" - -var ( - JarmHashRegex = regexp.MustCompile("(?m)0{62}") -) From 89cf7396b7a869e683db1e7aa35a2b17ad58841f Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 7 Jul 2022 09:34:31 -0500 Subject: [PATCH 214/355] modified error message with invalid ports (#677) * modified error message with invalid ports * updated ValidateCustomPorts function --- common/customports/customport.go | 31 +++++++++++++++++++++++++++++++ runner/options.go | 13 ++++++++++++- runner/runner.go | 4 ++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/common/customports/customport.go b/common/customports/customport.go index 8c5a025..5bd6029 100644 --- a/common/customports/customport.go +++ b/common/customports/customport.go @@ -1,6 +1,7 @@ package customport import ( + "fmt" "strconv" "strings" @@ -28,6 +29,36 @@ func (c *CustomPorts) String() string { return "Custom Ports" } +// ValidateCustomPorts to validate the custom port range +func ValidateCustomPorts(value string) error { + potentialRange := strings.Split(value, "-") + if len(potentialRange) < portRangeParts { + if _, err := strconv.Atoi(value); err != nil { + return fmt.Errorf("Could not cast port to integer from your value: %s. Resulting error: %s.\n", value, err.Error()) + } + } else { + part1 := potentialRange[0] + part2 := potentialRange[1] + lowP, err := strconv.Atoi(part1) + if err != nil { + return fmt.Errorf("Could not cast first port of your range(%s) to integer from your value: %s. Resulting error: %s.\n", + value, part1, err.Error()) + } + highP, err := strconv.Atoi(part2) + if err != nil { + return fmt.Errorf("Could not cast last port of your port range(%s) to integer from "+ + "your value: %s. Resulting error %s.\n", + value, part2, err.Error()) + } + if lowP > highP { + return fmt.Errorf("First value of port range should be lower than the last port "+ + "from your range: [%d, %d].\n", + lowP, highP) + } + } + return nil +} + // Set a port range func (c *CustomPorts) Set(value string) error { // ports can be like nmap -p [https|http:]start-end,[https|http:]port1,[https|http:]port2,[https|http:]port3 diff --git a/runner/options.go b/runner/options.go index a47db04..4e004d9 100644 --- a/runner/options.go +++ b/runner/options.go @@ -22,6 +22,7 @@ import ( fileutilz "github.com/projectdiscovery/httpx/common/fileutil" "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/httpx/common/stringz" + "github.com/projectdiscovery/sliceutil" ) const ( @@ -510,7 +511,17 @@ func (options *Options) ValidateOptions() error { if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 { options.OutputCDN = true } - + // Validate the custom port range + if len(options.CustomPorts) > 0 { + for _, value := range options.CustomPorts { + potentialPorts := sliceutil.Dedupe(strings.Split(value, ",")) + for _, port := range potentialPorts { + if err := customport.ValidateCustomPorts(port); err != nil { + return err + } + } + } + } return nil } diff --git a/runner/runner.go b/runner/runner.go index d046aa9..c28ca0c 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -562,6 +562,10 @@ func (r *Runner) RunEnumeration() { for resp := range output { if resp.err != nil { + // Change the error message if any port value passed explicitly + if url, err := url.Parse(resp.URL); err == nil && url.Port() != "" { + resp.err = errors.New(strings.ReplaceAll(resp.err.Error(), "address", "port")) + } gologger.Debug().Msgf("Failed '%s': %s\n", resp.URL, resp.err) } if resp.str == "" { From 3741669f693e37b979c95dd234960e48f153d89e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 20:08:19 +0530 Subject: [PATCH 215/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#683) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.48 to 0.0.49. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.48...v0.0.49) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5e18255..1638e17 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.48 + github.com/projectdiscovery/wappalyzergo v0.0.49 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 5ac01f2..ac29723 100644 --- a/go.sum +++ b/go.sum @@ -184,8 +184,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.48 h1:+5w6LzAdqYq3gJs4c5BkPYzOOqC7zYvA40ZoNLg/b+g= -github.com/projectdiscovery/wappalyzergo v0.0.48/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= +github.com/projectdiscovery/wappalyzergo v0.0.49 h1:h0ZQpAk+ypyKbVY/Mxy5B3OeQVS8+M0/Z/e4IJsqQwY= +github.com/projectdiscovery/wappalyzergo v0.0.49/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 59151fb6fb317c640724e5ef9ec1fcc7c000f70b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 20:08:33 +0530 Subject: [PATCH 216/355] chore(deps): bump github.com/microcosm-cc/bluemonday (#684) Bumps [github.com/microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday) from 1.0.18 to 1.0.19. - [Release notes](https://github.com/microcosm-cc/bluemonday/releases) - [Commits](https://github.com/microcosm-cc/bluemonday/compare/v1.0.18...v1.0.19) --- updated-dependencies: - dependency-name: github.com/microcosm-cc/bluemonday dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1638e17..3ff27be 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/microcosm-cc/bluemonday v1.0.18 + github.com/microcosm-cc/bluemonday v1.0.19 github.com/miekg/dns v1.1.48 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 diff --git a/go.sum b/go.sum index ac29723..82ef0ab 100644 --- a/go.sum +++ b/go.sum @@ -83,8 +83,9 @@ github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczG github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= -github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c= +github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= From 8d0a34f28a77136888448100901306cf7b1ea000 Mon Sep 17 00:00:00 2001 From: Its Hex ! Date: Sun, 10 Jul 2022 23:39:43 +0200 Subject: [PATCH 217/355] Update go.mod (#689) * Exposing regex matches to json output (#668) * Update runner.go Fix regex json output. * small refactor Co-authored-by: mzack * Update go.mod * fixing jarm-go reference * removing redundant declaration Co-authored-by: Sandeep Singh Co-authored-by: Dothelm <49758889+st3rv04ka@users.noreply.github.com> Co-authored-by: mzack --- common/hashes/jarmhash.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- runner/runner.go | 3 +++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/common/hashes/jarmhash.go b/common/hashes/jarmhash.go index 56c3936..d9fed77 100644 --- a/common/hashes/jarmhash.go +++ b/common/hashes/jarmhash.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/RumbleDiscovery/jarm-go" + "github.com/hdm/jarm-go" "golang.org/x/net/proxy" ) diff --git a/go.mod b/go.mod index 3ff27be..f300109 100644 --- a/go.mod +++ b/go.mod @@ -47,8 +47,8 @@ require github.com/spaolacci/murmur3 v1.1.0 require ( github.com/PuerkitoBio/goquery v1.8.0 - github.com/RumbleDiscovery/jarm-go v0.0.6 github.com/ammario/ipisp/v2 v2.0.0 + github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 ) diff --git a/go.sum b/go.sum index 82ef0ab..2d8f7c8 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb888350 github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= -github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= -github.com/RumbleDiscovery/jarm-go v0.0.6/go.mod h1:dXV7z5vBXQI0cNaHXwzGtq2PJ2LgM3XgcFiX32FU3bg= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= @@ -59,6 +57,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= +github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= +github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/runner/runner.go b/runner/runner.go index c28ca0c..37718c1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1291,6 +1291,7 @@ retry: } } + var extractRegex []string // extract regex var extractResult = map[string][]string{} if scanopts.extractRegexps != nil { @@ -1508,6 +1509,7 @@ retry: Lines: resp.Lines, Words: resp.Words, ASN: asnResponse, + ExtractRegex: extractRegex, } } @@ -1538,6 +1540,7 @@ type Result struct { CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty" csv:"tls-grab"` Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` + ExtractRegex []string `json:"extract-regex,omitempty" csv:"regex"` CDNName string `json:"cdn-name,omitempty" csv:"cdn-name"` Port string `json:"port,omitempty" csv:"port"` raw string From 84409dabd8f0b78ed8e456dd5e520c178f6c93c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 Jul 2022 21:40:17 +0000 Subject: [PATCH 218/355] chore(deps): bump github.com/projectdiscovery/mapcidr Bumps [github.com/projectdiscovery/mapcidr](https://github.com/projectdiscovery/mapcidr) from 1.0.0 to 1.0.1. - [Release notes](https://github.com/projectdiscovery/mapcidr/releases) - [Changelog](https://github.com/projectdiscovery/mapcidr/blob/master/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/mapcidr/compare/v1.0.0...v1.0.1) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/mapcidr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 9 ++++----- go.sum | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index f300109..a1913dd 100644 --- a/go.mod +++ b/go.mod @@ -20,15 +20,15 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.0.8-0.20220610073650-5d31a8c159e3 + github.com/projectdiscovery/goflags v0.0.8 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 - github.com/projectdiscovery/mapcidr v1.0.0 + github.com/projectdiscovery/mapcidr v1.0.1 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d - github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea + github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 github.com/projectdiscovery/wappalyzergo v0.0.49 @@ -38,7 +38,7 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20220607020251-c690dde0001d + golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 ) @@ -80,6 +80,5 @@ require ( golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2d8f7c8..1f3fcc1 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= -github.com/projectdiscovery/goflags v0.0.8-0.20220610073650-5d31a8c159e3 h1:2cUSicd6wkd7A3P+tjjAYiPmD99FtWp4k2curJkcrJ0= -github.com/projectdiscovery/goflags v0.0.8-0.20220610073650-5d31a8c159e3/go.mod h1:Np0EwqmopHDDb1cY4/NEBaC4fornMIWc6dr5yL91kzM= +github.com/projectdiscovery/goflags v0.0.8 h1:IhTmnEGSKtBHfD23tSwpnzai5CRYfLKVOWlVq9ngYvY= +github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -159,8 +159,9 @@ github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kA github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v1.0.0 h1:4ctYAgkxaLeky3o3c+qMgHoUjytYy6DME2fjbgiB1Ag= github.com/projectdiscovery/mapcidr v1.0.0/go.mod h1:5QkKrV6rNQQurCZI3nNedFsAOYp04mRDkC5yht+znYA= +github.com/projectdiscovery/mapcidr v1.0.1 h1:eaLBRrImwlYXv8vbXTwR4sxoQqIxR3Y5k/Sd7HhTIII= +github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= @@ -174,8 +175,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1 github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= -github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= -github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= +github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 h1:zZmLOwj2duDafEO9i1KqmHMW0SL9+Wwmco9sxhBCVwQ= +github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= @@ -216,8 +217,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -274,8 +276,8 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 0d554edd1bb02c4c109ad35f5cb3e383512e5a89 Mon Sep 17 00:00:00 2001 From: Dothelm <49758889+st3rv04ka@users.noreply.github.com> Date: Mon, 11 Jul 2022 01:42:21 +0400 Subject: [PATCH 219/355] Fix for -er and -favicon. (#687) * Exposing regex matches to json output (#668) * Update runner.go Fix regex json output. * small refactor Co-authored-by: mzack * Fix for -er and -favicon. Function don't clone regex for extract. * fixing errors * removing redundant code * removing unused field Co-authored-by: Sandeep Singh Co-authored-by: mzack --- runner/options.go | 1 + 1 file changed, 1 insertion(+) diff --git a/runner/options.go b/runner/options.go index 4e004d9..9b56d4b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -120,6 +120,7 @@ func (s *scanOptions) Clone() *scanOptions { MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead, HostMaxErrors: s.HostMaxErrors, Favicon: s.Favicon, + extractRegexps: s.extractRegexps, LeaveDefaultPorts: s.LeaveDefaultPorts, OutputLinesCount: s.OutputLinesCount, OutputWordsCount: s.OutputWordsCount, From 44938fdbc33a47b2df9534b34f52f1adf43af137 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Jul 2022 03:13:08 +0530 Subject: [PATCH 220/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo (#690) Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.49 to 0.0.50. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.49...v0.0.50) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f300109..039eee4 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.49 + github.com/projectdiscovery/wappalyzergo v0.0.50 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 2d8f7c8..c1122e2 100644 --- a/go.sum +++ b/go.sum @@ -185,8 +185,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.49 h1:h0ZQpAk+ypyKbVY/Mxy5B3OeQVS8+M0/Z/e4IJsqQwY= -github.com/projectdiscovery/wappalyzergo v0.0.49/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= +github.com/projectdiscovery/wappalyzergo v0.0.50 h1:mV0RwGnzyyfKscXnDCgzb2INy1BTJX8dT8Zurl+WCuY= +github.com/projectdiscovery/wappalyzergo v0.0.50/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 5e6b645a12e6bdc6f96062c15022cc01f9afb147 Mon Sep 17 00:00:00 2001 From: sandeep Date: Mon, 11 Jul 2022 03:57:11 +0530 Subject: [PATCH 221/355] readme update --- README.md | 12 +++++++----- runner/banner.go | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 30297a7..77f2e4a 100644 --- a/README.md +++ b/README.md @@ -116,11 +116,12 @@ MATCHERS: -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string match response with specified string (-ms admin) -mr, -match-regex string match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb) + -mcdn, -match-cdn string[] match host with specified cdn provider (azure, cloudflare, cloudfront, fastly, incapsula, oracle, google, akamai, sucuri, leaseweb) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') EXTRACTOR: - -er, -extract-regex string display response content for specified regex + -er, -extract-regex string[] Display response content with matched regex + -ep, -extract-preset string[] Display response content with matched preset regex FILTERS: -fc, -filter-code string filter response with specified status code (-fc 403,401) @@ -130,7 +131,7 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) -fs, -filter-string string filter response with specified string (-fs admin) -fe, -filter-regex string filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (azure, cloudflare, cloudfront, fastly, incapsula, oracle, google, akamai, sucuri, leaseweb) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') RATE-LIMIT: @@ -180,6 +181,7 @@ CONFIGURATIONS: -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443 DEBUG: + -health-check, -hc run diagnostic check up -debug display request/response content in cli -debug-req display request content in cli -debug-resp display response content in cli @@ -395,7 +397,7 @@ https://api.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20] ``` -### Path Probe +### File/Path Bruteforce ```console @@ -491,7 +493,7 @@ func main() { ``` -# 📋 Notes +# Notes - As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable. - For printing both HTTP/HTTPS results, `no-fallback` flag can be used. diff --git a/runner/banner.go b/runner/banner.go index 6c09f11..4107ce2 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.2 + /_/ v1.2.3 ` // Version is the current version of httpx -const Version = `v1.2.2` +const Version = `v1.2.3` // showBanner is used to show the banner to the user func showBanner() { From 7f8b37b8378108f303fd94b534c3dd1465e8521e Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 12 Jul 2022 19:37:20 +0200 Subject: [PATCH 222/355] updating goflags syntax --- runner/options.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/runner/options.go b/runner/options.go index 9b56d4b..3b6de66 100644 --- a/runner/options.go +++ b/runner/options.go @@ -221,10 +221,10 @@ type Options struct { Stream bool SkipDedupe bool ProbeAllIPS bool - Resolvers goflags.NormalizedStringSlice + Resolvers goflags.StringSlice Favicon bool - OutputFilterFavicon goflags.NormalizedStringSlice - OutputMatchFavicon goflags.NormalizedStringSlice + OutputFilterFavicon goflags.StringSlice + OutputMatchFavicon goflags.StringSlice LeaveDefaultPorts bool OutputLinesCount bool OutputMatchLinesCount string @@ -239,8 +239,8 @@ type Options struct { Hashes string Jarm bool Asn bool - OutputMatchCdn goflags.NormalizedStringSlice - OutputFilterCdn goflags.NormalizedStringSlice + OutputMatchCdn goflags.StringSlice + OutputFilterCdn goflags.StringSlice SniName string OutputMatchResponseTime string OutputFilterResponseTime string @@ -287,16 +287,16 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "match response with specified content length (-ml 100,102)"), flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "match response body with specified line count (-mlc 423,532)"), flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "match response body with specified word count (-mwc 43,55)"), - flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "match response with specified favicon hash (-mfc 1494302000)"), + flagSet.StringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", nil, "match response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), - flagSet.NormalizedStringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", []string{}, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders)), + flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"), ) flagSet.CreateGroup("extractor", "Extractor", - flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "Display response content with matched regex"), - flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, "Display response content with matched preset regex"), + flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "Display response content with matched regex", goflags.StringSliceOptions), + flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, "Display response content with matched preset regex", goflags.StringSliceOptions), ) flagSet.CreateGroup("filters", "Filters", @@ -304,10 +304,10 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "filter response with specified content length (-fl 23,33)"), flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "filter response body with specified line count (-flc 423,532)"), flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "filter response body with specified word count (-fwc 423,532)"), - flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "filter response with specified favicon hash (-mfc 1494302000)"), + flagSet.StringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", nil, "filter response with specified favicon hash (-mfc 1494302000)", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), - flagSet.NormalizedStringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", []string{}, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders)), + flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), ) @@ -341,7 +341,7 @@ func ParseOptions() *Options { ) flagSet.CreateGroup("configs", "Configurations", - flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "list of custom resolver (file or comma separated)"), + flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.NormalizedStringSliceOptions), flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "Custom TLS SNI name"), From 16c15aeb43450f7000593b7fc1ac2fb42bf44004 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 13 Jul 2022 19:34:54 +0200 Subject: [PATCH 223/355] Merging Master into dev-merge-master (#694) Exposing regex matches to json output (#668) * Update runner.go Fix regex json output. * small refactor Co-authored-by: mzack Co-authored-by: Sandeep Singh Co-authored-by: Dothelm <49758889+st3rv04ka@users.noreply.github.com> --- runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index 37718c1..e6b795e 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -23,7 +23,6 @@ import ( "time" "github.com/projectdiscovery/httpx/common/customextract" - "github.com/projectdiscovery/sliceutil" "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" @@ -34,6 +33,7 @@ import ( "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/sliceutil" "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" From af2a7947bc8ecb20543ad214d29666bafc82b9de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 10:09:29 +0000 Subject: [PATCH 224/355] chore(deps): bump golang from 1.18.3-alpine to 1.18.4-alpine Bumps golang from 1.18.3-alpine to 1.18.4-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b2e1ab5..70a81a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18.3-alpine AS builder +FROM golang:1.18.4-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 237de7b314bcf0f8c8d43de2a146735f44209fd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 10:14:09 +0000 Subject: [PATCH 225/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.50 to 0.0.51. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.50...v0.0.51) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6fd6045..a19ff69 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.50 + github.com/projectdiscovery/wappalyzergo v0.0.51 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index e421980..74b6a9c 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.50 h1:mV0RwGnzyyfKscXnDCgzb2INy1BTJX8dT8Zurl+WCuY= -github.com/projectdiscovery/wappalyzergo v0.0.50/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= +github.com/projectdiscovery/wappalyzergo v0.0.51 h1:/fieWDKZjF2JQy3ScKVEG1UVraqvoS4VbPGorgOhu08= +github.com/projectdiscovery/wappalyzergo v0.0.51/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 8f3ab2773e636e58c498b12e1187da1e50bc7f0d Mon Sep 17 00:00:00 2001 From: akkuman <1075768094@qq.com> Date: Fri, 22 Jul 2022 01:35:05 +0800 Subject: [PATCH 226/355] feat(runner): expose the method process (#700) feat(runner): expose the mothod process to support the pipeline pattern --- runner/runner.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/runner/runner.go b/runner/runner.go index e6b795e..d867da9 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -749,6 +749,14 @@ func (r *Runner) RunEnumeration() { wgoutput.Wait() } +func (r *Runner) GetScanOpts() scanOptions { + return r.scanopts +} + +func (r *Runner) Process(t string, wg *sizedwaitgroup.SizedWaitGroup, protocol string, scanopts *scanOptions, output chan Result) { + r.process(t, wg, r.hp, protocol, scanopts, output) +} + func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *scanOptions, output chan Result) { protocols := []string{protocol} if scanopts.NoFallback || protocol == httpx.HTTPandHTTPS { From 90a32dd65ff005f9d1f040885b59b6cd33f99286 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 25 Jul 2022 18:40:06 +0530 Subject: [PATCH 227/355] Added connection pooling for jarm conn + fastdialer (#707) Co-authored-by: Sandeep Singh --- common/hashes/jarm/connpool.go | 47 +++++++++++++++++ common/hashes/jarm/jarmhash.go | 81 ++++++++++++++++++++++++++++ common/hashes/jarm/onetimepool.go | 77 +++++++++++++++++++++++++++ common/hashes/jarmhash.go | 88 ------------------------------- go.mod | 1 + go.sum | 2 + runner/runner.go | 18 ++++++- 7 files changed, 225 insertions(+), 89 deletions(-) create mode 100644 common/hashes/jarm/connpool.go create mode 100644 common/hashes/jarm/jarmhash.go create mode 100644 common/hashes/jarm/onetimepool.go delete mode 100644 common/hashes/jarmhash.go diff --git a/common/hashes/jarm/connpool.go b/common/hashes/jarm/connpool.go new file mode 100644 index 0000000..56bc6b4 --- /dev/null +++ b/common/hashes/jarm/connpool.go @@ -0,0 +1,47 @@ +package jarm + +import ( + "net" + "sync" + + "go.uber.org/multierr" +) + +type inFlightConns struct { + sync.RWMutex + inflightConns map[net.Conn]struct{} +} + +func newInFlightConns() (*inFlightConns, error) { + return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil +} + +func (i *inFlightConns) Add(conn net.Conn) { + i.Lock() + defer i.Unlock() + + i.inflightConns[conn] = struct{}{} +} + +func (i *inFlightConns) Remove(conn net.Conn) { + i.Lock() + defer i.Unlock() + + delete(i.inflightConns, conn) +} + +func (i *inFlightConns) Close() error { + i.Lock() + defer i.Unlock() + + var errs []error + + for conn := range i.inflightConns { + if err := conn.Close(); err != nil { + errs = append(errs, err) + } + delete(i.inflightConns, conn) + } + + return multierr.Combine(errs...) +} diff --git a/common/hashes/jarm/jarmhash.go b/common/hashes/jarm/jarmhash.go new file mode 100644 index 0000000..ad75640 --- /dev/null +++ b/common/hashes/jarm/jarmhash.go @@ -0,0 +1,81 @@ +package jarm + +import ( + "context" + "fmt" + "net" + "net/url" + "strconv" + "strings" + "time" + + "github.com/hdm/jarm-go" + "github.com/projectdiscovery/fastdialer/fastdialer" +) + +const defaultPort int = 443 + +type target struct { + Host string + Port int +} + +// fingerprint probes a single host/port +func fingerprint(dialer *fastdialer.Dialer, t target, timeout time.Duration) string { + results := []string{} + addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port)) + // using connection pool as we need multiple probes + pool, err := newOneTimePool(context.Background(), addr, 3) + if err != nil { + return "" + } + pool.FastDialer = dialer + + defer pool.Close() //nolint + go pool.Run() //nolint + + for _, probe := range jarm.GetProbes(t.Host, t.Port) { + conn, err := pool.Acquire(context.Background()) + if err != nil { + continue + } + if conn == nil { + continue + } + _ = conn.SetWriteDeadline(time.Now().Add(timeout)) + _, err = conn.Write(jarm.BuildProbe(probe)) + if err != nil { + results = append(results, "") + _ = conn.Close() + continue + } + _ = conn.SetReadDeadline(time.Now().Add(timeout)) + buff := make([]byte, 1484) + _, _ = conn.Read(buff) + _ = conn.Close() + ans, err := jarm.ParseServerHello(buff, probe) + if err != nil { + results = append(results, "") + continue + } + results = append(results, ans) + } + return jarm.RawHashToFuzzyHash(strings.Join(results, ",")) +} + +func Jarm(dialer *fastdialer.Dialer, host string, duration int) string { + t := target{} + if u, err := url.Parse(host); err == nil { + if u.Scheme == "http" { + return "" + } + t.Host = u.Hostname() + port, _ := strconv.Atoi(u.Port()) + t.Port = port + } + if t.Port == 0 { + t.Port = defaultPort + } + timeout := time.Duration(duration) * time.Second + return fingerprint(dialer, t, timeout) +} diff --git a/common/hashes/jarm/onetimepool.go b/common/hashes/jarm/onetimepool.go new file mode 100644 index 0000000..c625ec8 --- /dev/null +++ b/common/hashes/jarm/onetimepool.go @@ -0,0 +1,77 @@ +package jarm + +import ( + "context" + "net" + + "github.com/projectdiscovery/fastdialer/fastdialer" +) + +// oneTimePool is a pool designed to create continous bare connections that are for one time only usage +type oneTimePool struct { + address string + idleConnections chan net.Conn + InFlightConns *inFlightConns + ctx context.Context + cancel context.CancelFunc + FastDialer *fastdialer.Dialer +} + +func newOneTimePool(ctx context.Context, address string, poolSize int) (*oneTimePool, error) { + idleConnections := make(chan net.Conn, poolSize) + inFlightConns, err := newInFlightConns() + if err != nil { + return nil, err + } + pool := &oneTimePool{ + address: address, + idleConnections: idleConnections, + InFlightConns: inFlightConns, + } + if ctx != nil { + pool.ctx = ctx + } + pool.ctx, pool.cancel = context.WithCancel(ctx) + return pool, nil +} + +// Acquire acquires an idle connection from the pool +func (p *oneTimePool) Acquire(c context.Context) (net.Conn, error) { + select { + case <-p.ctx.Done(): + return nil, p.ctx.Err() + case <-c.Done(): + return nil, c.Err() + case conn := <-p.idleConnections: + p.InFlightConns.Remove(conn) + return conn, nil + } +} + +func (p *oneTimePool) Run() error { + for { + select { + case <-p.ctx.Done(): + return p.ctx.Err() + default: + var ( + conn net.Conn + err error + ) + if p.FastDialer != nil { + conn, err = p.FastDialer.Dial(p.ctx, "tcp", p.address) + } else { + conn, err = net.Dial("tcp", p.address) + } + if err == nil { + p.InFlightConns.Add(conn) + p.idleConnections <- conn + } + } + } +} + +func (p *oneTimePool) Close() error { + p.cancel() + return p.InFlightConns.Close() +} diff --git a/common/hashes/jarmhash.go b/common/hashes/jarmhash.go deleted file mode 100644 index d9fed77..0000000 --- a/common/hashes/jarmhash.go +++ /dev/null @@ -1,88 +0,0 @@ -package hashes - -import ( - "fmt" - "net" - "net/url" - "strconv" - "strings" - "time" - - "github.com/hdm/jarm-go" - "golang.org/x/net/proxy" -) - -const defaultPort int = 443 - -var DefualtBackoff = func(r, m int) time.Duration { - return time.Second -} - -type target struct { - Host string - Port int - Retries int - Backoff func(r, m int) time.Duration -} - -// fingerprint probes a single host/port -func fingerprint(t target, duration int) string { - timeout := time.Duration(duration) * time.Second - results := []string{} - for _, probe := range jarm.GetProbes(t.Host, t.Port) { - dialer := proxy.FromEnvironmentUsing(&net.Dialer{Timeout: timeout}) - addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port)) - c := net.Conn(nil) - n := 0 - for c == nil && n <= t.Retries { - // Ignoring error since error message was already being dropped. - // Also, if theres an error, c == nil. - if c, _ = dialer.Dial("tcp", addr); c != nil || t.Retries == 0 { - break - } - bo := t.Backoff - if bo == nil { - bo = DefualtBackoff - } - time.Sleep(bo(n, t.Retries)) - n++ - } - if c == nil { - return "" - } - data := jarm.BuildProbe(probe) - _ = c.SetWriteDeadline(time.Now().Add(timeout)) - _, err := c.Write(data) - if err != nil { - results = append(results, "") - c.Close() - continue - } - _ = c.SetReadDeadline(time.Now().Add(timeout)) - buff := make([]byte, 1484) - _, _ = c.Read(buff) - c.Close() - ans, err := jarm.ParseServerHello(buff, probe) - if err != nil { - results = append(results, "") - continue - } - results = append(results, ans) - } - return jarm.RawHashToFuzzyHash(strings.Join(results, ",")) -} -func Jarm(host string, duration int) string { - t := target{} - if u, err := url.Parse(host); err == nil { - if u.Scheme == "http" { - return "" - } - t.Host = u.Hostname() - port, _ := strconv.Atoi(u.Port()) - t.Port = port - } - if t.Port == 0 { - t.Port = defaultPort - } - return fingerprint(t, duration) -} diff --git a/go.mod b/go.mod index a19ff69..a263a65 100644 --- a/go.mod +++ b/go.mod @@ -75,6 +75,7 @@ require ( github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect + go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect diff --git a/go.sum b/go.sum index 74b6a9c..e8d1d04 100644 --- a/go.sum +++ b/go.sum @@ -241,6 +241,8 @@ go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/runner/runner.go b/runner/runner.go index d867da9..b3a62d0 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -22,7 +22,9 @@ import ( "strings" "time" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" + "github.com/projectdiscovery/httpx/common/hashes/jarm" "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" @@ -62,6 +64,7 @@ type Runner struct { options *Options hp *httpx.HTTPX wappalyzer *wappalyzer.Wappalyze + fastdialer *fastdialer.Dialer scanopts scanOptions hm *hybrid.HybridMap stats clistats.StatisticsClient @@ -82,6 +85,19 @@ func New(options *Options) (*Runner, error) { return nil, errors.Wrap(err, "could not create wappalyzer client") } + dialerOpts := fastdialer.DefaultOptions + dialerOpts.WithDialerHistory = true + dialerOpts.MaxRetries = 3 + dialerOpts.DialerTimeout = time.Duration(options.Timeout) * time.Second + if len(options.Resolvers) > 0 { + dialerOpts.BaseResolvers = options.Resolvers + } + fastDialer, err := fastdialer.NewDialer(dialerOpts) + if err != nil { + return nil, errors.Wrap(err, "could not create dialer") + } + runner.fastdialer = fastDialer + httpxOptions := httpx.DefaultOptions // Enables automatically tlsgrab if tlsprobe is requested httpxOptions.TLSGrab = options.TLSGrab || options.TLSProbe @@ -1398,7 +1414,7 @@ retry: } jarmhash := "" if r.options.Jarm { - jarmhash = hashes.Jarm(fullURL, r.options.Timeout) + jarmhash = jarm.Jarm(r.fastdialer, fullURL, r.options.Timeout) builder.WriteString(" [") if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(jarmhash).String()) From fc9c73c134b116805d691b08e9dce97d13a0383d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 18:40:40 +0530 Subject: [PATCH 228/355] chore(deps): bump alpine from 3.16.0 to 3.16.1 (#708) Bumps alpine from 3.16.0 to 3.16.1. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 70a81a2..0133332 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.18.4-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.16.0 +FROM alpine:3.16.1 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From f40868aa60dddf8f7a824ca196000b0b53feb352 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 18:40:54 +0530 Subject: [PATCH 229/355] chore(deps): bump github.com/projectdiscovery/goflags from 0.0.8 to 0.0.9 (#709) chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.0.8 to 0.0.9. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.0.8...v0.0.9) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a263a65..f27bc27 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.0.8 + github.com/projectdiscovery/goflags v0.0.9 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 diff --git a/go.sum b/go.sum index e8d1d04..d4ad033 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,9 @@ github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= -github.com/projectdiscovery/goflags v0.0.8 h1:IhTmnEGSKtBHfD23tSwpnzai5CRYfLKVOWlVq9ngYvY= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= +github.com/projectdiscovery/goflags v0.0.9 h1:bPsYIPE1LvdgYaM3XNX0YmS68e6huv22W22rKh5IscI= +github.com/projectdiscovery/goflags v0.0.9/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= From f5f46669a3ad4746dd38d248c02dc4a8c9d634a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 18:41:05 +0530 Subject: [PATCH 230/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.51 to 0.0.52 (#710) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.51 to 0.0.52. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.51...v0.0.52) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f27bc27..9db0da3 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.51 + github.com/projectdiscovery/wappalyzergo v0.0.52 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index d4ad033..78006ce 100644 --- a/go.sum +++ b/go.sum @@ -187,8 +187,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.51 h1:/fieWDKZjF2JQy3ScKVEG1UVraqvoS4VbPGorgOhu08= -github.com/projectdiscovery/wappalyzergo v0.0.51/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= +github.com/projectdiscovery/wappalyzergo v0.0.52 h1:AbM+4KEikwgxpeoWg4Gf4KPxkXAeLHSpWyUaNcaW+sQ= +github.com/projectdiscovery/wappalyzergo v0.0.52/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 931c7e25bd5aa3c1c8917672e0a9c9a2770612cf Mon Sep 17 00:00:00 2001 From: Sajad Date: Tue, 26 Jul 2022 18:57:04 +0530 Subject: [PATCH 231/355] replace / by [slash] instead of underscore (#711) --- go.mod | 2 +- runner/runner.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9db0da3..5a389cd 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/ammario/ipisp/v2 v2.0.0 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 + go.uber.org/multierr v1.8.0 ) require ( @@ -75,7 +76,6 @@ require ( github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect - go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect diff --git a/runner/runner.go b/runner/runner.go index b3a62d0..19471b1 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1444,7 +1444,7 @@ retry: domainFile = domainFile[:maxFileNameLength] } - domainFile = strings.ReplaceAll(domainFile, "/", "_") + ".txt" + domainFile = strings.ReplaceAll(domainFile, "/", "[slash]") + ".txt" // store response responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile) respRaw := resp.Raw From 21ca2621bd702ab4367cc72eafd720bf3093a595 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Sun, 31 Jul 2022 17:24:34 +0200 Subject: [PATCH 232/355] Fixing and improving custom ports handling (#714) * Fixing and improving custom ports * adding test case from https://github.com/projectdiscovery/httpx/pull/713 --- cmd/functional-test/testcases.txt | 1 + common/customports/customport.go | 82 +++++++++++-------------------- runner/options.go | 15 +----- 3 files changed, 32 insertions(+), 66 deletions(-) diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index 0138562..476c901 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -14,3 +14,4 @@ scanme.sh {{binary}} -silent -unsafe scanme.sh {{binary}} -silent -x all scanme.sh {{binary}} -silent -body 'a=b' scanme.sh {{binary}} -silent -exclude-cdn +scanme.sh {{binary}} -silent -ports https:443 \ No newline at end of file diff --git a/common/customports/customport.go b/common/customports/customport.go index 5bd6029..60df7e0 100644 --- a/common/customports/customport.go +++ b/common/customports/customport.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" - "github.com/projectdiscovery/gologger" + "github.com/pkg/errors" "github.com/projectdiscovery/sliceutil" "github.com/projectdiscovery/httpx/common/httpx" @@ -29,36 +29,6 @@ func (c *CustomPorts) String() string { return "Custom Ports" } -// ValidateCustomPorts to validate the custom port range -func ValidateCustomPorts(value string) error { - potentialRange := strings.Split(value, "-") - if len(potentialRange) < portRangeParts { - if _, err := strconv.Atoi(value); err != nil { - return fmt.Errorf("Could not cast port to integer from your value: %s. Resulting error: %s.\n", value, err.Error()) - } - } else { - part1 := potentialRange[0] - part2 := potentialRange[1] - lowP, err := strconv.Atoi(part1) - if err != nil { - return fmt.Errorf("Could not cast first port of your range(%s) to integer from your value: %s. Resulting error: %s.\n", - value, part1, err.Error()) - } - highP, err := strconv.Atoi(part2) - if err != nil { - return fmt.Errorf("Could not cast last port of your port range(%s) to integer from "+ - "your value: %s. Resulting error %s.\n", - value, part2, err.Error()) - } - if lowP > highP { - return fmt.Errorf("First value of port range should be lower than the last port "+ - "from your range: [%d, %d].\n", - lowP, highP) - } - } - return nil -} - // Set a port range func (c *CustomPorts) Set(value string) error { // ports can be like nmap -p [https|http:]start-end,[https|http:]port1,[https|http:]port2,[https|http:]port3 @@ -83,39 +53,38 @@ func (c *CustomPorts) Set(value string) error { potentialRange := strings.Split(potentialPort, "-") // it's a single port? if len(potentialRange) < portRangeParts { - if p, err := strconv.Atoi(potentialPort); err == nil { - if existingProtocol, ok := Ports[p]; ok { - if existingProtocol == httpx.HTTP && protocol == httpx.HTTPS || existingProtocol == httpx.HTTPS && protocol == httpx.HTTP { - protocol = httpx.HTTPandHTTPS - } + p, err := strconv.Atoi(potentialPort) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Could not cast port to integer from your value: %s\n", potentialPort)) + } + if err := checkPortValue(p); err != nil { + return err + } + if existingProtocol, ok := Ports[p]; ok { + if existingProtocol == httpx.HTTP && protocol == httpx.HTTPS || existingProtocol == httpx.HTTPS && protocol == httpx.HTTP { + protocol = httpx.HTTPandHTTPS } - Ports[p] = protocol - } else { - gologger.Warning().Msgf("Could not cast port to integer from your value: %s. Resulting error: %s. Skipping it.\n", - potentialPort, err.Error()) } + Ports[p] = protocol } else { // expand range - var lowP, highP int lowP, err := strconv.Atoi(potentialRange[0]) if err != nil { - gologger.Warning().Msgf("Could not cast first port of your range(%s) to integer from your value: %s. Resulting error: %s. Skipping it.\n", - potentialPort, potentialRange[0], err.Error()) - continue + return errors.Wrap(err, fmt.Sprintf("Could not cast first port of your range(%s) to integer from your value: %s", potentialPort, potentialRange[0])) + } + if err := checkPortValue(lowP); err != nil { + return errors.Wrap(err, fmt.Sprintf("first port of your range(%d)", lowP)) } - highP, err = strconv.Atoi(potentialRange[1]) + highP, err := strconv.Atoi(potentialRange[1]) if err != nil { - gologger.Warning().Msgf("Could not cast last port of your port range(%s) to integer from "+ - "your value: %s. Resulting error %s. Skipping it\n", - potentialPort, potentialRange[1], err.Error()) - continue + return errors.Wrap(err, fmt.Sprintf("Could not cast last port of your port range(%s) to integer from your value: %s", potentialPort, potentialRange[1])) + } + if err := checkPortValue(highP); err != nil { + return errors.Wrap(err, fmt.Sprintf("last port of your range(%d)", lowP)) } if lowP > highP { - gologger.Warning().Msgf("First value of port range should be lower than the last port "+ - "from your range: [%d, %d]. Skipping it.\n", - lowP, highP) - continue + return fmt.Errorf("First value of port range should be lower than the last port from your range: [%d, %d]", lowP, highP) } for i := lowP; i <= highP; i++ { @@ -132,3 +101,10 @@ func (c *CustomPorts) Set(value string) error { *c = append(*c, value) return nil } + +func checkPortValue(port int) error { + if port > 65535 { + return errors.New("port value is bigger than 65535") + } + return nil +} diff --git a/runner/options.go b/runner/options.go index 3b6de66..53d7990 100644 --- a/runner/options.go +++ b/runner/options.go @@ -22,7 +22,6 @@ import ( fileutilz "github.com/projectdiscovery/httpx/common/fileutil" "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/httpx/common/stringz" - "github.com/projectdiscovery/sliceutil" ) const ( @@ -319,7 +318,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("Misc", "Miscellaneous", flagSet.BoolVarP(&options.ProbeAllIPS, "probe-all-ips", "pa", false, "probe all the ips associated with same host"), - flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg 1,2-10,11)"), + flagSet.VarP(&options.CustomPorts, "ports", "p", "ports to probe (nmap syntax: eg http:1,2-10,11,https:80)"), flagSet.StringVar(&options.RequestURIs, "path", "", "path or list of paths to probe (comma-separated, file)"), flagSet.BoolVar(&options.TLSProbe, "tls-probe", false, "send http probes on the extracted TLS domains (dns_name)"), flagSet.BoolVar(&options.CSPProbe, "csp-probe", false, "send http probes on the extracted CSP domains"), @@ -512,17 +511,7 @@ func (options *Options) ValidateOptions() error { if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 { options.OutputCDN = true } - // Validate the custom port range - if len(options.CustomPorts) > 0 { - for _, value := range options.CustomPorts { - potentialPorts := sliceutil.Dedupe(strings.Split(value, ",")) - for _, port := range potentialPorts { - if err := customport.ValidateCustomPorts(port); err != nil { - return err - } - } - } - } + return nil } From 10fe8a89b79d2d2298385ec58f511a8d3827eb2c Mon Sep 17 00:00:00 2001 From: sandeep Date: Sun, 31 Jul 2022 21:48:00 +0530 Subject: [PATCH 233/355] version update --- runner/banner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index 4107ce2..5f64df5 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.3 + /_/ v1.2.4 ` // Version is the current version of httpx -const Version = `v1.2.3` +const Version = `v1.2.4` // showBanner is used to show the banner to the user func showBanner() { From d796bb308c3dbda47aa402908b5ef33fb600b94b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 10:18:33 +0000 Subject: [PATCH 234/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.52 to 0.0.53. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.52...v0.0.53) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 5a389cd..26e372b 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.52 + github.com/projectdiscovery/wappalyzergo v0.0.53 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 78006ce..3a05caf 100644 --- a/go.sum +++ b/go.sum @@ -187,8 +187,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.52 h1:AbM+4KEikwgxpeoWg4Gf4KPxkXAeLHSpWyUaNcaW+sQ= -github.com/projectdiscovery/wappalyzergo v0.0.52/go.mod h1:1LQBGQVW47tMHxGTxmBK+pAwfsWKSLQMXt/egxGlljo= +github.com/projectdiscovery/wappalyzergo v0.0.53 h1:Ka3PdaZGsc0e/eNwZujs6H24yRxJsV8LKQ3AfPoXAlA= +github.com/projectdiscovery/wappalyzergo v0.0.53/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -218,7 +218,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= From 9cb577f71b5bc49f0a7e7d7d91c9826b75e7d7eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Aug 2022 10:11:05 +0000 Subject: [PATCH 235/355] chore(deps): bump golang from 1.18.4-alpine to 1.19.0-alpine Bumps golang from 1.18.4-alpine to 1.19.0-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0133332..711a977 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18.4-alpine AS builder +FROM golang:1.19.0-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From d11ac45ccc04b2320339dd0d2f75c8cbb65f016f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Aug 2022 10:21:02 +0000 Subject: [PATCH 236/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.53 to 0.0.54. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.53...v0.0.54) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 26e372b..a21ff10 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.53 + github.com/projectdiscovery/wappalyzergo v0.0.54 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3a05caf..a63f3b0 100644 --- a/go.sum +++ b/go.sum @@ -187,8 +187,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.53 h1:Ka3PdaZGsc0e/eNwZujs6H24yRxJsV8LKQ3AfPoXAlA= -github.com/projectdiscovery/wappalyzergo v0.0.53/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.54 h1:Qmr4LFNzFekwohxhH4SycvuL26tNzPU+O9NeFJ4FM8s= +github.com/projectdiscovery/wappalyzergo v0.0.54/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 8c10e307e415d6ae9c9e78608dfac427f9de7a77 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 11 Aug 2022 09:51:49 -0500 Subject: [PATCH 237/355] favicon flag improvement (#718) * favicon flag improvement * addressed review comments * utilizing request object --- runner/options.go | 5 ----- runner/runner.go | 18 +++++++++++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/runner/options.go b/runner/options.go index 53d7990..91252d7 100644 --- a/runner/options.go +++ b/runner/options.go @@ -496,11 +496,6 @@ func (options *Options) ValidateOptions() error { options.StoreResponse = true } - if options.Favicon { - gologger.Debug().Msgf("Setting single path to \"favicon.ico\" and ignoring multiple paths settings\n") - options.RequestURIs = "/favicon.ico" - } - if options.Hashes != "" { for _, hashType := range strings.Split(options.Hashes, ",") { if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) { diff --git a/runner/runner.go b/runner/runner.go index 19471b1..6613cca 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1343,17 +1343,21 @@ retry: } builder.WriteRune(']') } - var faviconMMH3 string if scanopts.Favicon { - faviconMMH3 = fmt.Sprintf("%d", stringz.FaviconHash(resp.Data)) - builder.WriteString(" [") - if !scanopts.OutputWithNoColor { - builder.WriteString(aurora.Magenta(faviconMMH3).String()) + req.URL.Path = "/favicon.ico" + if faviconResp, favErr := hp.Do(req, httpx.UnsafeOptions{}); favErr == nil { + faviconMMH3 = fmt.Sprintf("%d", stringz.FaviconHash(faviconResp.Data)) + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(faviconMMH3).String()) + } else { + builder.WriteString(faviconMMH3) + } + builder.WriteRune(']') } else { - builder.WriteString(faviconMMH3) + gologger.Warning().Msgf("Could not fetch favicon: %s", favErr.Error()) } - builder.WriteRune(']') } // adding default hashing for json output format if r.options.JSONOutput && len(scanopts.Hashes) == 0 { From c48909f64f8250e585ea7e7ea24891660fc82387 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 17 Aug 2022 02:16:06 +0300 Subject: [PATCH 238/355] Minor fixes and improvements (#726) * Added .idea to .gitignore * Updated the README * Added more verbose information on preset extractors * removing deprecated ioutil Co-authored-by: Mzack9999 --- .gitignore | 4 +-- README.md | 36 +++++++++++++-------------- cmd/integration-test/http.go | 6 ++--- common/customextract/customextract.go | 2 +- common/fileutil/fileutil.go | 4 +-- common/httputilz/httputilz.go | 4 +-- common/httpx/encodings.go | 8 +++--- common/httpx/http2.go | 3 +-- common/httpx/httpx.go | 3 +-- go.mod | 7 +++--- go.sum | 10 +++++--- runner/options.go | 16 ++++++------ runner/runner.go | 23 ++++++++++------- 13 files changed, 67 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index b8f8c16..028e9b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ - +.idea/ +.vscode/ cmd/httpx/httpx integration_tests/httpx integration_tests/integration-test @@ -6,4 +7,3 @@ cmd/functional-test/httpx_dev cmd/functional-test/functional-test cmd/functional-test/httpx cmd/functional-test/*.cfg -.vscode/ diff --git a/README.md b/README.md index 77f2e4a..1ada6ee 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@

-httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library, it is designed to maintain the result reliability with increased threads. +httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library. It is designed to maintain result reliability with an increased number of threads. # Features @@ -42,23 +42,23 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi ### Supported probes:- -| Probes | Default check | Probes | Default check | -| --------------- | ------------- | ----------------- | ------------- | -| URL | true | IP | true | -| Title | true | CNAME | true | -| Status Code | true | Raw HTTP | false | -| Content Length | true | HTTP2 | false | -| TLS Certificate | true | HTTP Pipeline | false | -| CSP Header | true | Virtual host | false | -| Line Count | true | Word Count | true | -| Location Header | true | CDN | false | -| Web Server | true | Paths | false | -| Web Socket | true | Ports | false | -| Response Time | true | Request Method | true | -| Favicon Hash | false | Probe Status | false | -| Body Hash | true | Header Hash | true | -| Redirect chain | false | URL Scheme | true | -| JARM Hash | false | ASN | false | +| Probes | Default check | Probes | Default check | +|-----------------|---------------|----------------|---------------| +| URL | true | IP | true | +| Title | true | CNAME | true | +| Status Code | true | Raw HTTP | false | +| Content Length | true | HTTP2 | false | +| TLS Certificate | true | HTTP Pipeline | false | +| CSP Header | true | Virtual host | false | +| Line Count | true | Word Count | true | +| Location Header | true | CDN | false | +| Web Server | true | Paths | false | +| Web Socket | true | Ports | false | +| Response Time | true | Request Method | true | +| Favicon Hash | false | Probe Status | false | +| Body Hash | true | Header Hash | true | +| Redirect chain | false | URL Scheme | true | +| JARM Hash | false | ASN | false | # Installation Instructions diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index bd0f6d8..72fd687 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strings" @@ -207,7 +207,7 @@ func (h *issue400) Execute() error { router := httprouter.New() router.POST("/receive", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") - data, _ := ioutil.ReadAll(r.Body) + data, _ := io.ReadAll(r.Body) fmt.Fprintf(w, "data received %s", data) })) ts = httptest.NewServer(router) @@ -231,7 +231,7 @@ func (h *issue414) Execute() error { router := httprouter.New() router.POST(uripath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { w.Header().Add("Content-Type", "application/json") - data, _ := ioutil.ReadAll(r.Body) + data, _ := io.ReadAll(r.Body) fmt.Fprintf(w, "data received %s", data) })) ts = httptest.NewServer(router) diff --git a/common/customextract/customextract.go b/common/customextract/customextract.go index d15afca..b3966d7 100644 --- a/common/customextract/customextract.go +++ b/common/customextract/customextract.go @@ -4,6 +4,6 @@ import "regexp" var ExtractPresets = map[string]*regexp.Regexp{ "url": regexp.MustCompile("^(http(s)?:\\/\\/)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+(:[0-9]{1,5})?[-a-zA-Z0-9()@:%_\\\\\\+\\.~#?&//=]*$"), //nolint - "ip": regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}"), //nolint + "ipv4": regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}"), //nolint "mail": regexp.MustCompile("^([A-Za-z0-9_\\-\\.\u4e00-\u9fa5])+\\@([A-Za-z0-9_\\-\\.])+\\.([A-Za-z]{2,8})$"), //nolint } diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go index 8fe5c6e..76213cb 100644 --- a/common/fileutil/fileutil.go +++ b/common/fileutil/fileutil.go @@ -3,7 +3,7 @@ package fileutil import ( "bufio" "errors" - "io/ioutil" + "net" "os" "path/filepath" @@ -72,7 +72,7 @@ func LoadCidrsFromSliceOrFileWithMaxRecursion(option string, splitchar string, m } else if _, _, err := net.ParseCIDR(item); err == nil { networkList = append(networkList, item) } else if fileutil.FileExists(item) { - if filedata, err := ioutil.ReadFile(item); err == nil && len(filedata) > 0 { + if filedata, err := os.ReadFile(item); err == nil && len(filedata) > 0 { networkList = append(networkList, LoadCidrsFromSliceOrFileWithMaxRecursion(string(filedata), "\n", maxRecursion-1)...) } } diff --git a/common/httputilz/httputilz.go b/common/httputilz/httputilz.go index 8b6367c..1aaa6fc 100644 --- a/common/httputilz/httputilz.go +++ b/common/httputilz/httputilz.go @@ -3,7 +3,7 @@ package httputilz import ( "bufio" "fmt" - "io/ioutil" + "io" "net/http/httputil" "strings" @@ -87,7 +87,7 @@ func ParseRequest(req string, unsafe bool) (method, path string, headers map[str } // Set the request body - b, err := ioutil.ReadAll(reader) + b, err := io.ReadAll(reader) if err != nil { err = fmt.Errorf("could not read request body: %s", err) return diff --git a/common/httpx/encodings.go b/common/httpx/encodings.go index 752e2e8..9c5114b 100644 --- a/common/httpx/encodings.go +++ b/common/httpx/encodings.go @@ -2,7 +2,7 @@ package httpx import ( "bytes" - "io/ioutil" + "io" "net/http" "strings" @@ -19,7 +19,7 @@ import ( func Decodegbk(s []byte) ([]byte, error) { I := bytes.NewReader(s) O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder()) - d, e := ioutil.ReadAll(O) + d, e := io.ReadAll(O) if e != nil { return nil, e } @@ -30,7 +30,7 @@ func Decodegbk(s []byte) ([]byte, error) { func Decodebig5(s []byte) ([]byte, error) { I := bytes.NewReader(s) O := transform.NewReader(I, traditionalchinese.Big5.NewDecoder()) - d, e := ioutil.ReadAll(O) + d, e := io.ReadAll(O) if e != nil { return nil, e } @@ -41,7 +41,7 @@ func Decodebig5(s []byte) ([]byte, error) { func Encodebig5(s []byte) ([]byte, error) { I := bytes.NewReader(s) O := transform.NewReader(I, traditionalchinese.Big5.NewEncoder()) - d, e := ioutil.ReadAll(O) + d, e := io.ReadAll(O) if e != nil { return nil, e } diff --git a/common/httpx/http2.go b/common/httpx/http2.go index a1f0bb3..9d1a705 100644 --- a/common/httpx/http2.go +++ b/common/httpx/http2.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "github.com/projectdiscovery/gologger" @@ -68,7 +67,7 @@ func (h *HTTPX) SupportHTTP2(protocol, method, targetURL string) bool { } func freeHTTPResources(response *http.Response) error { - _, err := io.Copy(ioutil.Discard, response.Body) + _, err := io.Copy(io.Discard, response.Body) if err != nil { return fmt.Errorf("could not discard response body: %s", err) } diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 4438e6f..c524ade 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -4,7 +4,6 @@ import ( "crypto/tls" "fmt" "io" - "io/ioutil" "net/http" "net/url" "strconv" @@ -202,7 +201,7 @@ get_response: // websockets don't have a readable body if httpresp.StatusCode != http.StatusSwitchingProtocols { var err error - respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) + respbody, err = io.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead)) if err != nil && !shouldIgnoreBodyErrors { return nil, err } diff --git a/go.mod b/go.mod index a21ff10..159b7ee 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/httpx -go 1.17 +go 1.18 require ( github.com/akrylysov/pogreb v0.10.1 // indirect @@ -51,6 +51,7 @@ require ( github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 go.uber.org/multierr v1.8.0 + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e ) require ( @@ -77,8 +78,8 @@ require ( github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect - golang.org/x/mod v0.4.2 // indirect - golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index a63f3b0..5fe1ae7 100644 --- a/go.sum +++ b/go.sum @@ -140,7 +140,6 @@ github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= -github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.0.9 h1:bPsYIPE1LvdgYaM3XNX0YmS68e6huv22W22rKh5IscI= github.com/projectdiscovery/goflags v0.0.9/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= @@ -254,10 +253,13 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -276,7 +278,6 @@ golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -327,8 +328,9 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/runner/options.go b/runner/options.go index 91252d7..920cf13 100644 --- a/runner/options.go +++ b/runner/options.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/pkg/errors" + "golang.org/x/exp/maps" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fileutil" @@ -16,6 +17,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/customheader" "github.com/projectdiscovery/httpx/common/customlist" customport "github.com/projectdiscovery/httpx/common/customports" @@ -251,7 +253,7 @@ func ParseOptions() *Options { options := &Options{} flagSet := goflags.NewFlagSet() - flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers using retryablehttp library.`) + flagSet.SetDescription(`httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library.`) flagSet.CreateGroup("input", "Input", flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"), @@ -294,8 +296,8 @@ func ParseOptions() *Options { ) flagSet.CreateGroup("extractor", "Extractor", - flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "Display response content with matched regex", goflags.StringSliceOptions), - flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, "Display response content with matched preset regex", goflags.StringSliceOptions), + flagSet.StringSliceVarP(&options.OutputExtractRegexs, "extract-regex", "er", nil, "display response content with matched regex", goflags.StringSliceOptions), + flagSet.StringSliceVarP(&options.OutputExtractPresets, "extract-preset", "ep", nil, fmt.Sprintf("display response content matched by a pre-defined regex (%s)", strings.Join(maps.Keys(customextract.ExtractPresets), ",")), goflags.StringSliceOptions), ) flagSet.CreateGroup("filters", "Filters", @@ -343,8 +345,8 @@ func ParseOptions() *Options { flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.NormalizedStringSliceOptions), flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), - flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "Custom TLS SNI name"), - flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), + flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "custom TLS SNI name"), + flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "enable Random User-Agent to use"), flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"), flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"), flagSet.BoolVar(&options.Unsafe, "unsafe", false, "send raw requests skipping golang normalization"), @@ -419,11 +421,11 @@ func ParseOptions() *Options { func (options *Options) ValidateOptions() error { if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { - return fmt.Errorf("File %s does not exist.", options.InputFile) + return fmt.Errorf("File '%s' does not exist.", options.InputFile) } if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) { - return fmt.Errorf("File %s does not exist.", options.InputRawRequest) + return fmt.Errorf("File '%s' does not exist.", options.InputRawRequest) } multiOutput := options.CSVOutput && options.JSONOutput diff --git a/runner/runner.go b/runner/runner.go index 6613cca..26bbbb6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -7,7 +7,7 @@ import ( "encoding/csv" "encoding/json" "fmt" - "io/ioutil" + "io" "net" "net/http" "net/http/httputil" @@ -22,6 +22,8 @@ import ( "strings" "time" + "golang.org/x/exp/maps" + "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" @@ -30,6 +32,7 @@ import ( "github.com/bluele/gcache" "github.com/logrusorgru/aurora" "github.com/pkg/errors" + "github.com/projectdiscovery/clistats" "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/goconfig" @@ -39,6 +42,9 @@ import ( "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" + "github.com/remeh/sizedwaitgroup" + "go.uber.org/ratelimit" + // automatic fd max increase if running as root _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/fileutil" @@ -55,8 +61,6 @@ import ( "github.com/projectdiscovery/mapcidr" "github.com/projectdiscovery/rawhttp" wappalyzer "github.com/projectdiscovery/wappalyzergo" - "github.com/remeh/sizedwaitgroup" - "go.uber.org/ratelimit" ) // Runner is a client for running the enumeration process. @@ -155,7 +159,7 @@ func New(options *Options) (*Runner, error) { if options.InputRawRequest != "" { var rawRequest []byte - rawRequest, err = ioutil.ReadFile(options.InputRawRequest) + rawRequest, err = os.ReadFile(options.InputRawRequest) if err != nil { gologger.Fatal().Msgf("Could not read raw request from path '%s': %s\n", options.InputRawRequest, err) } @@ -248,7 +252,8 @@ func New(options *Options) (*Runner, error) { if regex, ok := customextract.ExtractPresets[regexName]; ok { scanopts.extractRegexps[regexName] = regex } else { - gologger.Warning().Msgf("Could not find preset: %s\n", regexName) + availablePresets := strings.Join(maps.Keys(customextract.ExtractPresets), ",") + gologger.Warning().Msgf("Could not find preset: '%s'. Available presets are: %s\n", regexName, availablePresets) } } } @@ -983,7 +988,7 @@ retry: // We set content-length even if zero to allow net/http to follow 307/308 redirects (it fails on unknown size) if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) - req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) } else { req.ContentLength = 0 req.Body = nil @@ -1010,7 +1015,7 @@ retry: // Create a copy on the fly of the request body if scanopts.RequestBody != "" { req.ContentLength = int64(len(scanopts.RequestBody)) - req.Body = ioutil.NopCloser(strings.NewReader(scanopts.RequestBody)) + req.Body = io.NopCloser(strings.NewReader(scanopts.RequestBody)) } var errDump error requestDump, errDump = httputil.DumpRequestOut(req.Request, true) @@ -1455,14 +1460,14 @@ retry: if len(respRaw) > scanopts.MaxResponseBodySizeToSave { respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave] } - writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644) + writeErr := os.WriteFile(responsePath, []byte(respRaw), 0644) if writeErr != nil { gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } if scanopts.StoreChain && resp.HasChain() { domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile) - writeErr := ioutil.WriteFile(responsePath, []byte(resp.GetChain()), 0644) + writeErr := os.WriteFile(responsePath, []byte(resp.GetChain()), 0644) if writeErr != nil { gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } From 182f74666d7dc5dd4513f3e41e36fe51ede7e7b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Aug 2022 03:16:21 +0400 Subject: [PATCH 239/355] chore(deps): bump alpine from 3.16.1 to 3.16.2 (#727) Bumps alpine from 3.16.1 to 3.16.2. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 711a977..d2217da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.19.0-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.16.1 +FROM alpine:3.16.2 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From fff1cc19f1e740817d42082fb7e3f5562f017f58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Aug 2022 03:16:42 +0400 Subject: [PATCH 240/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.54 to 0.0.55 (#728) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.54 to 0.0.55. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.54...v0.0.55) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sandeep Singh --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 159b7ee..a9dc19a 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.54 + github.com/projectdiscovery/wappalyzergo v0.0.55 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 5fe1ae7..1808e7f 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.54 h1:Qmr4LFNzFekwohxhH4SycvuL26tNzPU+O9NeFJ4FM8s= -github.com/projectdiscovery/wappalyzergo v0.0.54/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.55 h1:dDWuohTrAUWrphnFB+uAL33QrDWBbdM+z7sTzSxy8PY= +github.com/projectdiscovery/wappalyzergo v0.0.55/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 2654ffa1076c09f7edc8c1c5fa9f5f32ef83bdc7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 16:04:28 +0530 Subject: [PATCH 241/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.55 to 0.0.56 (#731) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.55 to 0.0.56. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.55...v0.0.56) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a9dc19a..77dc7c5 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.55 + github.com/projectdiscovery/wappalyzergo v0.0.56 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1808e7f..d33accc 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.55 h1:dDWuohTrAUWrphnFB+uAL33QrDWBbdM+z7sTzSxy8PY= -github.com/projectdiscovery/wappalyzergo v0.0.55/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.56 h1:774LsI8tAUAYROQhTX9VsxoQN2kAC5m9CcNj3BHsvTs= +github.com/projectdiscovery/wappalyzergo v0.0.56/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From dce70eaedbb7750016edb0c431cab4246ed23791 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 15:47:34 +0530 Subject: [PATCH 242/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.56 to 0.0.57 (#742) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.56 to 0.0.57. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.56...v0.0.57) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 77dc7c5..2788f32 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.56 + github.com/projectdiscovery/wappalyzergo v0.0.57 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index d33accc..fac1774 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.56 h1:774LsI8tAUAYROQhTX9VsxoQN2kAC5m9CcNj3BHsvTs= -github.com/projectdiscovery/wappalyzergo v0.0.56/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.57 h1:/QDYw6qEZfTelMzRaTuhIVSVImmUJlvizCHWo/MTeU4= +github.com/projectdiscovery/wappalyzergo v0.0.57/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From e236531eac60245a4d6bd8f3f1500c415504fc7e Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 30 Aug 2022 17:05:14 +0530 Subject: [PATCH 243/355] Added profile-mem flag for memory profiling (#738) --- cmd/httpx/httpx.go | 20 ++++++++++++++++++++ runner/options.go | 2 ++ 2 files changed, 22 insertions(+) diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 8ddf3ce..6890365 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -3,6 +3,8 @@ package main import ( "os" "os/signal" + "runtime" + "runtime/pprof" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/httpx/runner" @@ -12,6 +14,24 @@ func main() { // Parse the command line flags and read config files options := runner.ParseOptions() + // Profiling related code + if options.Memprofile != "" { + f, err := os.Create(options.Memprofile) + if err != nil { + gologger.Fatal().Msgf("profile: could not create memory profile %q: %v", options.Memprofile, err) + } + old := runtime.MemProfileRate + runtime.MemProfileRate = 4096 + gologger.Print().Msgf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, options.Memprofile) + + defer func() { + _ = pprof.Lookup("heap").WriteTo(f, 0) + f.Close() + runtime.MemProfileRate = old + gologger.Print().Msgf("profile: memory profiling disabled, %s", options.Memprofile) + }() + } + httpxRunner, err := runner.New(options) if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) diff --git a/runner/options.go b/runner/options.go index 920cf13..7f1641b 100644 --- a/runner/options.go +++ b/runner/options.go @@ -231,6 +231,7 @@ type Options struct { OutputMatchLinesCount string matchLinesCount []int OutputFilterLinesCount string + Memprofile string filterLinesCount []int OutputWordsCount bool OutputMatchWordsCount string @@ -369,6 +370,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.DebugResponse, "debug-resp", false, "display response content in cli"), flagSet.BoolVar(&options.Version, "version", false, "display httpx version"), flagSet.BoolVar(&options.ShowStatistics, "stats", false, "display scan statistic"), + flagSet.StringVar(&options.Memprofile, "profile-mem", "", "optional httpx memory profile dump file"), flagSet.BoolVar(&options.Silent, "silent", false, "silent mode"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "verbose mode"), flagSet.IntVarP(&options.StatsInterval, "stats-interval", "si", 0, "number of seconds to wait between showing a statistics update (default: 5)"), From d04150075e152839ece71bf3dfdc855b91dc95fb Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Wed, 31 Aug 2022 13:22:26 +0530 Subject: [PATCH 244/355] JSON field update (#748) Misc changes to json fields --- runner/runner.go | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 26bbbb6..3848ca5 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1398,8 +1398,8 @@ retry: hashHeader = hashes.Simhash([]byte(resp.RawHeaders)) } if hashBody != "" { - hashesMap[fmt.Sprintf("body-%s", hashType)] = hashBody - hashesMap[fmt.Sprintf("header-%s", hashType)] = hashHeader + hashesMap[fmt.Sprintf("body_%s", hashType)] = hashBody + hashesMap[fmt.Sprintf("header_%s", hashType)] = hashHeader if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(hashBody).String()) } else { @@ -1571,10 +1571,10 @@ type Result struct { ASN interface{} `json:"asn,omitempty" csv:"asn"` err error CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty" csv:"tls-grab"` - Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` - ExtractRegex []string `json:"extract-regex,omitempty" csv:"regex"` - CDNName string `json:"cdn-name,omitempty" csv:"cdn-name"` + TLSData *cryptoutil.TLSData `json:"tls,omitempty" csv:"tls"` + Hashes map[string]string `json:"hash,omitempty" csv:"hash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` Port string `json:"port,omitempty" csv:"port"` raw string URL string `json:"url,omitempty" csv:"url"` @@ -1585,27 +1585,27 @@ type Result struct { Scheme string `json:"scheme,omitempty" csv:"scheme"` Error string `json:"error,omitempty" csv:"error"` WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"response-body,omitempty" csv:"response-body"` - ContentType string `json:"content-type,omitempty" csv:"content-type"` + ResponseBody string `json:"body,omitempty" csv:"body"` + ContentType string `json:"content_type,omitempty" csv:"content_type"` Method string `json:"method,omitempty" csv:"method"` Host string `json:"host,omitempty" csv:"host"` Path string `json:"path,omitempty" csv:"path"` - FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` - FinalURL string `json:"final-url,omitempty" csv:"final-url"` - ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` + FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` + FinalURL string `json:"final_url,omitempty" csv:"final_url"` + ResponseHeader string `json:"header,omitempty" csv:"header"` Request string `json:"request,omitempty" csv:"request"` - ResponseTime string `json:"response-time,omitempty" csv:"response-time"` + ResponseTime string `json:"time,omitempty" csv:"time"` Jarm string `json:"jarm,omitempty" csv:"jarm"` - ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` + ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` A []string `json:"a,omitempty" csv:"a"` - CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` - Technologies []string `json:"technologies,omitempty" csv:"technologies"` + CNAMEs []string `json:"cname,omitempty" csv:"cname"` + Technologies []string `json:"tech,omitempty" csv:"tech"` Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` Words int `json:"words" csv:"words"` Lines int `json:"lines" csv:"lines"` - StatusCode int `json:"status-code,omitempty" csv:"status-code"` - ContentLength int `json:"content-length,omitempty" csv:"content-length"` + StatusCode int `json:"status_code,omitempty" csv:"status_code"` + ContentLength int `json:"content_length,omitempty" csv:"content_length"` Failed bool `json:"failed" csv:"failed"` VHost bool `json:"vhost,omitempty" csv:"vhost"` WebSocket bool `json:"websocket,omitempty" csv:"websocket"` From 8f2b89095fd18ac30260c386cce9e6d6b5237496 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 01:13:43 +0530 Subject: [PATCH 245/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.57 to 0.0.58 (#755) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.57 to 0.0.58. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.57...v0.0.58) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2788f32..01f78f1 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.57 + github.com/projectdiscovery/wappalyzergo v0.0.58 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index fac1774..9815fb8 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.57 h1:/QDYw6qEZfTelMzRaTuhIVSVImmUJlvizCHWo/MTeU4= -github.com/projectdiscovery/wappalyzergo v0.0.57/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.58 h1:kLbaiIK5KqLC/imGna5OOs0DWjTWFnj7iqRajTmflc8= +github.com/projectdiscovery/wappalyzergo v0.0.58/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From f40cbe78ba81d2d04105af8c968df485069d3b4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Sep 2022 01:14:38 +0530 Subject: [PATCH 246/355] chore(deps): bump github.com/microcosm-cc/bluemonday from 1.0.19 to 1.0.20 (#756) chore(deps): bump github.com/microcosm-cc/bluemonday Bumps [github.com/microcosm-cc/bluemonday](https://github.com/microcosm-cc/bluemonday) from 1.0.19 to 1.0.20. - [Release notes](https://github.com/microcosm-cc/bluemonday/releases) - [Commits](https://github.com/microcosm-cc/bluemonday/compare/v1.0.19...v1.0.20) --- updated-dependencies: - dependency-name: github.com/microcosm-cc/bluemonday dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 01f78f1..4610856 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/microcosm-cc/bluemonday v1.0.19 + github.com/microcosm-cc/bluemonday v1.0.20 github.com/miekg/dns v1.1.48 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 @@ -38,8 +38,8 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect + golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/text v0.3.7 ) diff --git a/go.sum b/go.sum index 9815fb8..d54b0ce 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= -github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c= -github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= +github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= +github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= @@ -279,8 +279,8 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -313,8 +313,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From d42c929aab8d00685629f2a39df27113f4780cce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 04:31:40 +0530 Subject: [PATCH 247/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.58 to 0.0.59 (#762) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.58 to 0.0.59. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.58...v0.0.59) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4610856..10654a9 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.58 + github.com/projectdiscovery/wappalyzergo v0.0.59 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index d54b0ce..67eb0fa 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:Eb github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.58 h1:kLbaiIK5KqLC/imGna5OOs0DWjTWFnj7iqRajTmflc8= -github.com/projectdiscovery/wappalyzergo v0.0.58/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.59 h1:jwgmee27eF6OGrhq0ieHSvU+/0H9JVKQx0xDkpJYBTs= +github.com/projectdiscovery/wappalyzergo v0.0.59/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= From 065716c75820af92123249a5f5c1b41207574730 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Sep 2022 04:31:51 +0530 Subject: [PATCH 248/355] chore(deps): bump golang from 1.19.0-alpine to 1.19.1-alpine (#761) Bumps golang from 1.19.0-alpine to 1.19.1-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d2217da..e596015 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.0-alpine AS builder +FROM golang:1.19.1-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From d491201eff6caf265c574e607f5a64332282e1f1 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Tue, 13 Sep 2022 06:16:38 -0500 Subject: [PATCH 249/355] Issue 735+issue 739 (#757) * adding strings.ToLower to headers * added validation logic for flags * validated silent flag with incompatible flags * silent flag bug fix * added test case * updated test case to include multiple headers --- cmd/integration-test/http.go | 28 ++++++++++++++++++++++++++ common/customheader/customheader.go | 3 ++- internal/testutils/integration.go | 19 ++++++++++++++++++ runner/options.go | 31 +++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 72fd687..4951995 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -27,6 +27,7 @@ var httpTestcases = map[string]testutils.TestCase{ "Request URI to existing file - https://github.com/projectdiscovery/httpx/issues/480": &issue480{}, // request uri pointing to existing file "Standard HTTP GET Request with match response time": &standardHttpGet{mrt: true, inputValue: "\"<10s\""}, "Standard HTTP GET Request with filter response time": &standardHttpGet{frt: true, inputValue: "\">3ms\""}, + "Multiple Custom Header": &customHeader{inputData: []string{"-debug-req", "-H", "'user-agent: test'", "-H", "'foo: bar'"}, expectedOutput: []string{"User-Agent: test", "Foo: bar"}}, } type standardHttpGet struct { @@ -297,3 +298,30 @@ func (h *issue480) Execute() error { } return nil } + +type customHeader struct { + inputData []string + expectedOutput []string +} + +func (h *customHeader) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + fmt.Fprint(w, `{"status": "ok"}`) + })) + ts = httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetCombinedResults(ts.URL, true, h.inputData...) + if err != nil { + return err + } + for _, expected := range h.expectedOutput { + if !strings.Contains(results, expected) { + return errIncorrectResult(expected, results) + } + } + return nil +} diff --git a/common/customheader/customheader.go b/common/customheader/customheader.go index 8d17652..a739b00 100644 --- a/common/customheader/customheader.go +++ b/common/customheader/customheader.go @@ -1,6 +1,7 @@ package customheader import ( + "strings" "github.com/projectdiscovery/stringsutil" ) @@ -21,7 +22,7 @@ func (c *CustomHeaders) Set(value string) error { // Has checks if the list contains a header name func (c *CustomHeaders) Has(header string) bool { for _, customHeader := range *c { - if stringsutil.HasPrefixAny(customHeader, header) { + if stringsutil.HasPrefixAny(strings.ToLower(customHeader), strings.ToLower(header)) { return true } } diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go index d058952..8b5af02 100644 --- a/internal/testutils/integration.go +++ b/internal/testutils/integration.go @@ -34,6 +34,25 @@ func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, e } return parts, nil } + +// RunNucleiAndGetResults returns a list of results for a template +func RunHttpxAndGetCombinedResults(url string, debug bool, extra ...string) (string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := `echo "` + url + `" | ./httpx ` + cmdLine += strings.Join(extra, " ") + if debug { + cmdLine += " -debug" + } else { + cmdLine += " -silent" + } + + cmd.Args = append(cmd.Args, cmdLine) + data, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + return string(data), nil +} func RunHttpxBinaryAndGetResults(target string, httpxBinary string, debug bool, args []string) ([]string, error) { cmd := exec.Command("bash", "-c") cmdLine := fmt.Sprintf(`echo %s | %s `, target, httpxBinary) diff --git a/runner/options.go b/runner/options.go index 7f1641b..c815d96 100644 --- a/runner/options.go +++ b/runner/options.go @@ -435,6 +435,22 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("Results can only be displayed in one format: 'JSON' or 'CSV'") } + if options.Silent { + incompatibleFlagsList := flagsIncompatibleWithSilent(options) + if len(incompatibleFlagsList) > 0 { + last := incompatibleFlagsList[len(incompatibleFlagsList)-1] + first := incompatibleFlagsList[:len(incompatibleFlagsList)-1] + msg := "" + if len(incompatibleFlagsList) > 1 { + msg += fmt.Sprintf("%s and %s flags are", strings.Join(first, ", "), last) + } else { + msg += fmt.Sprintf("%s flag is", last) + } + msg += " incompatible with silent flag" + return fmt.Errorf(msg) + } + } + var err error if options.matchStatusCode, err = stringz.StringToSliceInt(options.OutputMatchStatusCode); err != nil { return errors.Wrap(err, "Invalid value for match status code option") @@ -552,3 +568,18 @@ func (options *Options) ShouldLoadResume() bool { func (options *Options) ShouldSaveResume() bool { return true } + +func flagsIncompatibleWithSilent(options *Options) []string { + var incompatibleFlagsList []string + for k, v := range map[string]bool{ + "debug": options.Debug, + "debug-request": options.DebugRequests, + "debug-response": options.DebugResponse, + "verbose": options.Verbose, + } { + if v { + incompatibleFlagsList = append(incompatibleFlagsList, k) + } + } + return incompatibleFlagsList +} From 085cec481348cf585f83c36d344f7dce19b4ee7e Mon Sep 17 00:00:00 2001 From: Ice3man Date: Wed, 14 Sep 2022 12:44:36 +0530 Subject: [PATCH 250/355] Header field update for JSON output (#766) --- runner/runner.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 3848ca5..ad62735 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1195,11 +1195,13 @@ retry: var serverResponseRaw string var request string - var responseHeader string + var rawResponseHeader string + var responseHeader map[string]string if scanopts.ResponseInStdout { serverResponseRaw = string(resp.Data) request = string(requestDump) - responseHeader = resp.RawHeaders + responseHeader = normalizeHeaders(resp.Headers) + rawResponseHeader = resp.RawHeaders } // check for virtual host @@ -1504,6 +1506,7 @@ retry: Timestamp: time.Now(), Request: request, ResponseHeader: responseHeader, + RawHeader: rawResponseHeader, Scheme: parsed.Scheme, Port: finalPort, Path: finalPath, @@ -1592,7 +1595,8 @@ type Result struct { Path string `json:"path,omitempty" csv:"path"` FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` FinalURL string `json:"final_url,omitempty" csv:"final_url"` - ResponseHeader string `json:"header,omitempty" csv:"header"` + ResponseHeader map[string]string `json:"header,omitempty" csv:"header"` + RawHeader string `json:"raw_header,omitempty" csv:"raw_header"` Request string `json:"request,omitempty" csv:"request"` ResponseTime string `json:"time,omitempty" csv:"time"` Jarm string `json:"jarm,omitempty" csv:"jarm"` @@ -1726,3 +1730,11 @@ func getDNSData(hp *httpx.HTTPX, hostname string) (ips, cnames []string, err err cnames = dnsData.CNAME return } + +func normalizeHeaders(headers map[string][]string) map[string]string { + normalized := make(map[string]string, len(headers)) + for k, v := range headers { + normalized[strings.ReplaceAll(strings.ToLower(k), "-", "_")] = strings.Join(v, ", ") + } + return normalized +} From 9b4a2ecb0f5cc2f53d10059aa8cd204887e311a8 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Thu, 15 Sep 2022 20:33:51 +0530 Subject: [PATCH 251/355] Added tlsx integration for probing and grabbing (#767) * Added tlsx integration for probing and grabbing * Added additional TLS required fields --- common/httpx/response.go | 4 +- common/httpx/tls.go | 69 ++++++++++- go.mod | 30 ++++- go.sum | 250 +++++++++++++++++++++++++++++++++++++-- runner/runner.go | 32 ++--- 5 files changed, 344 insertions(+), 41 deletions(-) diff --git a/common/httpx/response.go b/common/httpx/response.go index 99f6333..fa561f7 100644 --- a/common/httpx/response.go +++ b/common/httpx/response.go @@ -4,8 +4,8 @@ import ( "strings" "time" - "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/httputil" + "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" ) // Response contains the response to a server @@ -18,7 +18,7 @@ type Response struct { RawHeaders string Words int Lines int - TLSData *cryptoutil.TLSData + TLSData *clients.Response CSPData *CSPData HTTP2 bool Pipeline bool diff --git a/common/httpx/tls.go b/common/httpx/tls.go index aa90c98..cd7fd15 100644 --- a/common/httpx/tls.go +++ b/common/httpx/tls.go @@ -1,15 +1,74 @@ package httpx import ( + "crypto/tls" + "crypto/x509" + "net" "net/http" - "github.com/projectdiscovery/cryptoutil" + "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" ) +// versionToTLSVersionString converts tls version to version string +var versionToTLSVersionString = map[uint16]string{ + tls.VersionTLS10: "tls10", + tls.VersionTLS11: "tls11", + tls.VersionTLS12: "tls12", + tls.VersionTLS13: "tls13", +} + // TLSGrab fills the TLSData -func (h *HTTPX) TLSGrab(r *http.Response) *cryptoutil.TLSData { - if r.TLS != nil { - return cryptoutil.TLSGrab(r.TLS) +func (h *HTTPX) TLSGrab(r *http.Response) *clients.Response { + if r.TLS == nil || len(r.TLS.PeerCertificates) == 0 { + return nil + } + host := r.Request.URL.Host + hostname, port, _ := net.SplitHostPort(host) + if hostname == "" { + hostname = host + } + if port == "" { + port = "443" + } + + tlsVersion := versionToTLSVersionString[r.TLS.Version] + tlsCipher := tls.CipherSuiteName(r.TLS.CipherSuite) + + leafCertificate := r.TLS.PeerCertificates[0] + response := &clients.Response{ + Host: hostname, + ProbeStatus: true, + Port: port, + Version: tlsVersion, + Cipher: tlsCipher, + TLSConnection: "ctls", + CertificateResponse: convertCertificateToResponse(hostname, leafCertificate), + ServerName: r.TLS.ServerName, + } + return response +} + +func convertCertificateToResponse(hostname string, cert *x509.Certificate) *clients.CertificateResponse { + response := &clients.CertificateResponse{ + SubjectAN: cert.DNSNames, + Emails: cert.EmailAddresses, + NotBefore: cert.NotBefore, + NotAfter: cert.NotAfter, + Expired: clients.IsExpired(cert.NotAfter), + SelfSigned: clients.IsSelfSigned(cert.AuthorityKeyId, cert.SubjectKeyId), + MisMatched: clients.IsMisMatchedCert(hostname, append(cert.DNSNames, cert.Subject.CommonName)), + WildCardCert: clients.IsWildCardCert(append(cert.DNSNames, cert.Subject.CommonName)), + IssuerCN: cert.Issuer.CommonName, + IssuerOrg: cert.Issuer.Organization, + SubjectCN: cert.Subject.CommonName, + SubjectOrg: cert.Subject.Organization, + FingerprintHash: clients.CertificateResponseFingerprintHash{ + MD5: clients.MD5Fingerprint(cert.Raw), + SHA1: clients.SHA1Fingerprint(cert.Raw), + SHA256: clients.SHA256Fingerprint(cert.Raw), + }, } - return nil + response.IssuerDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawIssuer, cert.Issuer.String()) + response.SubjectDN = clients.ParseASN1DNSequenceWithZpkixOrDefault(cert.RawSubject, cert.Subject.String()) + return response } diff --git a/go.mod b/go.mod index 10654a9..cf8616d 100644 --- a/go.mod +++ b/go.mod @@ -11,18 +11,18 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.20 - github.com/miekg/dns v1.1.48 // indirect + github.com/miekg/dns v1.1.50 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 + github.com/projectdiscovery/fastdialer v0.0.16-0.20220620143737-2ba20b53770a github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.0.9 github.com/projectdiscovery/gologger v1.1.4 - github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa + github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 github.com/projectdiscovery/mapcidr v1.0.1 @@ -50,37 +50,57 @@ require ( github.com/ammario/ipisp/v2 v2.0.0 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 + github.com/projectdiscovery/tlsx v0.0.7 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e ) require ( + github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/cockroachdb/errors v1.8.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect + github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae // indirect + github.com/cockroachdb/redact v1.0.8 // indirect + github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect + github.com/dgraph-io/badger v1.6.2 // indirect + github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gogo/protobuf v1.3.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.11.7 // indirect + github.com/kr/pretty v0.3.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.13 // indirect + github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect - github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 // indirect + github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect - github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect + github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c // indirect golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 67eb0fa..ac3b183 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,26 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= @@ -12,27 +30,96 @@ github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9or github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae h1:rH4i31f3O0yEQy1Ho6DuUqf0+0Pq4MIflc8TBZEMvUI= +github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= +github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -40,37 +127,69 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -78,11 +197,22 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= @@ -90,8 +220,10 @@ github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= -github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -99,13 +231,18 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -114,7 +251,11 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -128,12 +269,13 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 h1:uxOMKHOajVaiz/pMUzc3nvWhICa/euZN+SoVzpqcPiE= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220620143737-2ba20b53770a h1:LiekfR8hqK3bWr9aE3VU3gPofdBgN4OVvJkIVejlSYM= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220620143737-2ba20b53770a/go.mod h1:AJ2E9V04tk1jMQ6ffgqudfSoO2rcQvM7L9Uc6a9UmmQ= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483/go.mod h1:wjS/oBWbzlayJ/aTK0KW0oOHGO03G8oEYzuN6stI8Ho= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c h1:/+xJK8e/Gj/zAmudWh5l2SzGJB+CkwYnraelCkBe7Aw= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= @@ -146,8 +288,9 @@ github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLY github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= -github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa h1:9sZWFUAshIa/ea0RKjGRuuZiS5PzYXAFjTRUnSbezr0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= +github.com/projectdiscovery/hmap v0.0.2 h1:fe3k0b6tj95mn9a1phD3JXvAAOmmWAh/Upg1Bf0Tfos= +github.com/projectdiscovery/hmap v0.0.2/go.mod h1:YU3TeNTDmLW2dtb4LvuEtDsPTsQ06XMnmOeD3KOuU6c= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= @@ -158,7 +301,6 @@ github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619/go.mod h1: github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= github.com/projectdiscovery/mapcidr v1.0.0/go.mod h1:5QkKrV6rNQQurCZI3nNedFsAOYp04mRDkC5yht+znYA= github.com/projectdiscovery/mapcidr v1.0.1 h1:eaLBRrImwlYXv8vbXTwR4sxoQqIxR3Y5k/Sd7HhTIII= github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= @@ -180,14 +322,17 @@ github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:EbP+rVR7NPsFKfwhzshmXP2GMjqBpaMZurJzQ5PYvbY= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= +github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgtkDavCk= +github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= github.com/projectdiscovery/wappalyzergo v0.0.59 h1:jwgmee27eF6OGrhq0ieHSvU+/0H9JVKQx0xDkpJYBTs= github.com/projectdiscovery/wappalyzergo v0.0.59/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -195,8 +340,13 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -204,8 +354,15 @@ github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsR github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -221,19 +378,36 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= -github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95 h1:DyAZOw3JsVd6LJHqhl4MpKQdYQEmat0C6pPPwom39Ow= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= +github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 h1:vMJBP3PQViZsF6cOINtvyMC8ptpLsyJ4EwyFnzuWNxc= +github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b h1:iYQzlljG1dOXBtsJGyzFC/wBK5qUCWs1eLCr/UcJYPA= github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw= +github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c h1:ufDm/IlBYZYLuiqvQuhpTKwrcAS2OlXEzWbDvTVGbSQ= +github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c/go.mod h1:egdRkzUylATvPkWMpebZbXhv0FMEMJGX/ur0D3Csk2s= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= @@ -245,7 +419,9 @@ go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -253,17 +429,34 @@ golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -281,23 +474,34 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -319,14 +523,23 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= @@ -336,6 +549,16 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -344,15 +567,20 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -365,3 +593,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/runner/runner.go b/runner/runner.go index ad62735..8167620 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -27,6 +27,7 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" + "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" @@ -34,7 +35,6 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/clistats" - "github.com/projectdiscovery/cryptoutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/retryablehttp-go" @@ -796,17 +796,14 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false - for _, tt := range result.TLSData.DNSNames { + for _, tt := range result.TLSData.SubjectAN { if !r.testAndSet(tt) { continue } r.process(tt, wg, hp, protocol, scanopts, output) } - for _, tt := range result.TLSData.CommonName { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) + if r.testAndSet(result.TLSData.SubjectCN) { + r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) } } if scanopts.CSPProbe && result.CSPData != nil { @@ -838,17 +835,14 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false - for _, tt := range result.TLSData.DNSNames { + for _, tt := range result.TLSData.SubjectAN { if !r.testAndSet(tt) { continue } r.process(tt, wg, hp, protocol, scanopts, output) } - for _, tt := range result.TLSData.CommonName { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) + if r.testAndSet(result.TLSData.SubjectCN) { + r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) } } }(port, target, method, wantedProtocol) @@ -1573,12 +1567,12 @@ type Result struct { Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` ASN interface{} `json:"asn,omitempty" csv:"asn"` err error - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - TLSData *cryptoutil.TLSData `json:"tls,omitempty" csv:"tls"` - Hashes map[string]string `json:"hash,omitempty" csv:"hash"` - ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` - CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` - Port string `json:"port,omitempty" csv:"port"` + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` + TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` + Hashes map[string]string `json:"hash,omitempty" csv:"hash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` + Port string `json:"port,omitempty" csv:"port"` raw string URL string `json:"url,omitempty" csv:"url"` Input string `json:"input,omitempty" csv:"input"` From 674b792770de9ca7b0b8d6c598feeeddbd63828e Mon Sep 17 00:00:00 2001 From: Ice3man Date: Thu, 15 Sep 2022 22:57:26 +0530 Subject: [PATCH 252/355] Fixed connection pooling with cancelable timeout (#763) --- common/hashes/jarm/connpool.go | 47 ------------------------------- common/hashes/jarm/jarmhash.go | 30 +++++++++++--------- common/hashes/jarm/onetimepool.go | 39 +++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 60 deletions(-) delete mode 100644 common/hashes/jarm/connpool.go diff --git a/common/hashes/jarm/connpool.go b/common/hashes/jarm/connpool.go deleted file mode 100644 index 56bc6b4..0000000 --- a/common/hashes/jarm/connpool.go +++ /dev/null @@ -1,47 +0,0 @@ -package jarm - -import ( - "net" - "sync" - - "go.uber.org/multierr" -) - -type inFlightConns struct { - sync.RWMutex - inflightConns map[net.Conn]struct{} -} - -func newInFlightConns() (*inFlightConns, error) { - return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil -} - -func (i *inFlightConns) Add(conn net.Conn) { - i.Lock() - defer i.Unlock() - - i.inflightConns[conn] = struct{}{} -} - -func (i *inFlightConns) Remove(conn net.Conn) { - i.Lock() - defer i.Unlock() - - delete(i.inflightConns, conn) -} - -func (i *inFlightConns) Close() error { - i.Lock() - defer i.Unlock() - - var errs []error - - for conn := range i.inflightConns { - if err := conn.Close(); err != nil { - errs = append(errs, err) - } - delete(i.inflightConns, conn) - } - - return multierr.Combine(errs...) -} diff --git a/common/hashes/jarm/jarmhash.go b/common/hashes/jarm/jarmhash.go index ad75640..5b45656 100644 --- a/common/hashes/jarm/jarmhash.go +++ b/common/hashes/jarm/jarmhash.go @@ -13,7 +13,10 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" ) -const defaultPort int = 443 +const ( + poolCount = 3 + defaultPort = 443 +) type target struct { Host string @@ -21,11 +24,15 @@ type target struct { } // fingerprint probes a single host/port -func fingerprint(dialer *fastdialer.Dialer, t target, timeout time.Duration) string { +func fingerprint(dialer *fastdialer.Dialer, t target, duration int) string { results := []string{} addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port)) - // using connection pool as we need multiple probes - pool, err := newOneTimePool(context.Background(), addr, 3) + timeout := time.Duration(duration) * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(duration*poolCount) * time.Second)) + defer cancel() + + pool, err := newOneTimePool(ctx, addr, poolCount) if err != nil { return "" } @@ -35,19 +42,18 @@ func fingerprint(dialer *fastdialer.Dialer, t target, timeout time.Duration) str go pool.Run() //nolint for _, probe := range jarm.GetProbes(t.Host, t.Port) { - conn, err := pool.Acquire(context.Background()) + conn, err := pool.Acquire(ctx) if err != nil { - continue + return "" } if conn == nil { - continue + return "" } _ = conn.SetWriteDeadline(time.Now().Add(timeout)) _, err = conn.Write(jarm.BuildProbe(probe)) if err != nil { - results = append(results, "") _ = conn.Close() - continue + return "" } _ = conn.SetReadDeadline(time.Now().Add(timeout)) buff := make([]byte, 1484) @@ -55,8 +61,7 @@ func fingerprint(dialer *fastdialer.Dialer, t target, timeout time.Duration) str _ = conn.Close() ans, err := jarm.ParseServerHello(buff, probe) if err != nil { - results = append(results, "") - continue + return "" } results = append(results, ans) } @@ -76,6 +81,5 @@ func Jarm(dialer *fastdialer.Dialer, host string, duration int) string { if t.Port == 0 { t.Port = defaultPort } - timeout := time.Duration(duration) * time.Second - return fingerprint(dialer, t, timeout) + return fingerprint(dialer, t, duration) } diff --git a/common/hashes/jarm/onetimepool.go b/common/hashes/jarm/onetimepool.go index c625ec8..a30fe6e 100644 --- a/common/hashes/jarm/onetimepool.go +++ b/common/hashes/jarm/onetimepool.go @@ -3,8 +3,10 @@ package jarm import ( "context" "net" + "sync" "github.com/projectdiscovery/fastdialer/fastdialer" + "go.uber.org/multierr" ) // oneTimePool is a pool designed to create continous bare connections that are for one time only usage @@ -75,3 +77,40 @@ func (p *oneTimePool) Close() error { p.cancel() return p.InFlightConns.Close() } + +type inFlightConns struct { + sync.Mutex + inflightConns map[net.Conn]struct{} +} + +func newInFlightConns() (*inFlightConns, error) { + return &inFlightConns{inflightConns: make(map[net.Conn]struct{})}, nil +} + +func (i *inFlightConns) Add(conn net.Conn) { + i.Lock() + defer i.Unlock() + + i.inflightConns[conn] = struct{}{} +} + +func (i *inFlightConns) Remove(conn net.Conn) { + i.Lock() + defer i.Unlock() + + delete(i.inflightConns, conn) +} + +func (i *inFlightConns) Close() error { + i.Lock() + defer i.Unlock() + + var errs []error + for conn := range i.inflightConns { + if err := conn.Close(); err != nil { + errs = append(errs, err) + } + delete(i.inflightConns, conn) + } + return multierr.Combine(errs...) +} From a2daa0f3b89e2a3963a6e9c4b48e7898f8854eb4 Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Fri, 16 Sep 2022 12:23:11 +0530 Subject: [PATCH 253/355] Update dependencies version (#770) * chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.0.9 to 0.1.0. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.0.9...v0.1.0) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore(deps): bump github.com/projectdiscovery/mapcidr Bumps [github.com/projectdiscovery/mapcidr](https://github.com/projectdiscovery/mapcidr) from 1.0.1 to 1.0.2. - [Release notes](https://github.com/projectdiscovery/mapcidr/releases) - [Changelog](https://github.com/projectdiscovery/mapcidr/blob/master/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/mapcidr/compare/v1.0.1...v1.0.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/mapcidr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * resolve the dependency issue [WIP] * Update dependencies version - dependabot PR is failing due to version mismatch * Revert "Merge branch 'dev' of https://github.com/projectdiscovery/httpx into dependency_version_change" This reverts commit 4634851eecf602004dc6360b5759a11c952b436d, reversing changes made to b7800f906c257db8ec3af5a620d8135349a56d7b. * Revert "Revert "Merge branch 'dev' of https://github.com/projectdiscovery/httpx into dependency_version_change"" This reverts commit eec0fa69e547d9d4cecba9e94df9476f493f8a89. * revert to merge from dev branch - run `go mod tidy` to add missing packages Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- go.mod | 31 +++++++++++------------ go.sum | 79 +++++++++++++++++++++++++++++++--------------------------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/go.mod b/go.mod index cf8616d..a84ee28 100644 --- a/go.mod +++ b/go.mod @@ -15,21 +15,21 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 - github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.16-0.20220620143737-2ba20b53770a + github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect + github.com/projectdiscovery/fastdialer v0.0.16 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c + github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.0.9 + github.com/projectdiscovery/goflags v0.1.0 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 - github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 - github.com/projectdiscovery/mapcidr v1.0.1 + github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 + github.com/projectdiscovery/mapcidr v1.0.2 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d - github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 - github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 + github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 + github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 github.com/projectdiscovery/wappalyzergo v0.0.59 github.com/remeh/sizedwaitgroup v1.0.0 @@ -38,8 +38,8 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/net v0.0.0-20220909164309-bea034e7d591 + golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect golang.org/x/text v0.3.7 ) @@ -84,10 +84,10 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect - github.com/projectdiscovery/networkpolicy v0.0.1 // indirect + github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect + github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect - github.com/projectdiscovery/retryabledns v1.0.13 // indirect + github.com/projectdiscovery/retryabledns v1.0.15 // indirect github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect @@ -97,9 +97,8 @@ require ( github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c // indirect golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/tools v0.1.10 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/tools v0.1.12 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index ac3b183..d5dd46c 100644 --- a/go.sum +++ b/go.sum @@ -214,12 +214,10 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvG github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= -github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -261,29 +259,28 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= -github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d h1:1Q94movG4VMVo+qf781ttXXt2bxBOIGMenmuCAJDLfE= -github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= +github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK68yPOnNNRmwuXqytl+T9EbwneEUCvMDRjLe0J04= +github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220620143737-2ba20b53770a h1:LiekfR8hqK3bWr9aE3VU3gPofdBgN4OVvJkIVejlSYM= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220620143737-2ba20b53770a/go.mod h1:AJ2E9V04tk1jMQ6ffgqudfSoO2rcQvM7L9Uc6a9UmmQ= +github.com/projectdiscovery/fastdialer v0.0.16 h1:0FcWVAtm6ftVALQpckdlKTeZcJi277HduvGXsBDO02Q= +github.com/projectdiscovery/fastdialer v0.0.16/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= -github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= -github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483/go.mod h1:wjS/oBWbzlayJ/aTK0KW0oOHGO03G8oEYzuN6stI8Ho= -github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c h1:/+xJK8e/Gj/zAmudWh5l2SzGJB+CkwYnraelCkBe7Aw= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= +github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 h1:4o97N9ftX1J3iKlIRVMPVOVZs4qbCczJvoFF2WA40t4= +github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= -github.com/projectdiscovery/goflags v0.0.9 h1:bPsYIPE1LvdgYaM3XNX0YmS68e6huv22W22rKh5IscI= -github.com/projectdiscovery/goflags v0.0.9/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI= +github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= +github.com/projectdiscovery/goflags v0.1.0 h1:Z7sUVK8wgH6aGJWinmGQEtsn+GNf/0RQ+z1wQcpCeeA= +github.com/projectdiscovery/goflags v0.1.0/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -295,37 +292,37 @@ github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:Gzruq github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= -github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU= -github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 h1:PSYGmqVDxBe9Pnbe14MIX7NXTHJqxMwBLzwDNBwSZYk= -github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619/go.mod h1:w5bXH/PHqM8/vF0bFN6fZMggm7IlPKK2dJdz5McY+Qw= +github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 h1:HRqev12wKvcwK1fe4pSlMfQdPHo9LfTxuFeRN4f3tS4= +github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1:vHRC+9exsfSbEngMKDl0xiWqkxlLk3lHQZpbS2yFT8U= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= -github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v1.0.0/go.mod h1:5QkKrV6rNQQurCZI3nNedFsAOYp04mRDkC5yht+znYA= -github.com/projectdiscovery/mapcidr v1.0.1 h1:eaLBRrImwlYXv8vbXTwR4sxoQqIxR3Y5k/Sd7HhTIII= github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= -github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= +github.com/projectdiscovery/mapcidr v1.0.2 h1:ewn7hoZz8P1TJ4VqG//XMlQKg5zhj2hnEx3C9jf3sQA= +github.com/projectdiscovery/mapcidr v1.0.2/go.mod h1:9OuUDjA+FilmXToex0xMfQb0os6qMaoiuZCksai0asY= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= +github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= +github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= -github.com/projectdiscovery/retryabledns v1.0.13 h1:Ogfv0fl3Nszb+Nq2S2qQmE+PJDlStVxyLkmTotMohZY= -github.com/projectdiscovery/retryabledns v1.0.13/go.mod h1:EeqHcAPp0g2GljT4qkxKSAE47Dj0ZrJQ46R9ct3Muhk= +github.com/projectdiscovery/retryabledns v1.0.15 h1:3Nn119UwYsfUPC3g0q57ftz0Wb5Zl5ppvw8R0Xu0DEI= +github.com/projectdiscovery/retryabledns v1.0.15/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= -github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 h1:zZmLOwj2duDafEO9i1KqmHMW0SL9+Wwmco9sxhBCVwQ= github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA= +github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= +github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:EbP+rVR7NPsFKfwhzshmXP2GMjqBpaMZurJzQ5PYvbY= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= +github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= +github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ= +github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM= github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgtkDavCk= github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= @@ -374,6 +371,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= @@ -402,6 +400,7 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDf github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= @@ -427,6 +426,7 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -444,8 +444,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -471,9 +471,12 @@ golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -481,8 +484,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -509,16 +513,17 @@ golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= +golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -542,13 +547,13 @@ golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= From f8dc25cc066913dd17991b10810b0716bbc99476 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Sep 2022 15:59:59 +0530 Subject: [PATCH 254/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.59 to 0.0.60 (#777) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.59 to 0.0.60. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.59...v0.0.60) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a84ee28..6c6543d 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.59 + github.com/projectdiscovery/wappalyzergo v0.0.60 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index d5dd46c..ab89102 100644 --- a/go.sum +++ b/go.sum @@ -327,8 +327,8 @@ github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgt github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.59 h1:jwgmee27eF6OGrhq0ieHSvU+/0H9JVKQx0xDkpJYBTs= -github.com/projectdiscovery/wappalyzergo v0.0.59/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.60 h1:xD1l+LrW3LLY7OlsKYtizEwfv7gnttF/w8at8ZHJ2s0= +github.com/projectdiscovery/wappalyzergo v0.0.60/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= From c90b897eba37280955a5d9a36a08aeafcb60b2c0 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Mon, 19 Sep 2022 12:20:40 -0500 Subject: [PATCH 255/355] list-dsl flag to list json output keys which can be further used with matchers / filters (#768) * flag list-dsl * variable name update * used reflect to dnymically access DSL variables * aligned with the changes in dev branch Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- common/httpx/csp.go | 2 +- runner/options.go | 11 +++++- runner/runner.go | 62 ----------------------------- runner/types.go | 95 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 65 deletions(-) create mode 100644 runner/types.go diff --git a/common/httpx/csp.go b/common/httpx/csp.go index e1c4831..d663baf 100644 --- a/common/httpx/csp.go +++ b/common/httpx/csp.go @@ -19,7 +19,7 @@ var CSPHeaders = []string{ // CSPData contains the Content-Security-Policy domain list type CSPData struct { - Domains []string `json:"domains,omitempty"` + Domains []string `json:"domains,omitempty" dsl:"domains"` } // CSPGrab fills the CSPData diff --git a/runner/options.go b/runner/options.go index c815d96..afb7138 100644 --- a/runner/options.go +++ b/runner/options.go @@ -247,6 +247,7 @@ type Options struct { OutputMatchResponseTime string OutputFilterResponseTime string HealthCheck bool + ListDSLVariable bool } // ParseOptions parses the command line options for application @@ -329,6 +330,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Pipeline, "pipeline", false, "probe and display server supporting HTTP1.1 pipeline"), flagSet.BoolVar(&options.HTTP2Probe, "http2", false, "probe and display server supporting HTTP2"), flagSet.BoolVar(&options.VHost, "vhost", false, "probe and display server supporting VHOST"), + flagSet.BoolVarP(&options.ListDSLVariable, "list-dsl-variables", "ldv", false, "list json output field keys name that support dsl matcher/filter"), ) flagSet.CreateGroup("output", "Output", @@ -406,7 +408,12 @@ func ParseOptions() *Options { if err != nil { gologger.Fatal().Msgf("%s\n", err) } - + if options.ListDSLVariable { + for _, dsl := range dslVariables() { + gologger.Print().Msg(dsl) + } + os.Exit(0) + } showBanner() if options.Version { @@ -450,7 +457,7 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf(msg) } } - + var err error if options.matchStatusCode, err = stringz.StringToSliceInt(options.OutputMatchStatusCode); err != nil { return errors.Wrap(err, "Invalid value for match status code option") diff --git a/runner/runner.go b/runner/runner.go index 8167620..7d90302 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -27,7 +27,6 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" - "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" @@ -1551,67 +1550,6 @@ func (r *Runner) SaveResumeConfig() error { return goconfig.Save(resumeCfg, DefaultResumeFile) } -type AsnResponse struct { - AsNumber string `json:"as-number" csv:"as-number"` - AsName string `json:"as-name" csv:"as-name"` - AsCountry string `json:"as-country" csv:"as-country"` - AsRange string `json:"as-range" csv:"as-range"` -} - -func (o AsnResponse) String() string { - return fmt.Sprintf("%v, %v, %v, %v", o.AsNumber, o.AsName, o.AsCountry, o.AsRange) -} - -// Result of a scan -type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` - ASN interface{} `json:"asn,omitempty" csv:"asn"` - err error - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` - Hashes map[string]string `json:"hash,omitempty" csv:"hash"` - ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` - CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` - Port string `json:"port,omitempty" csv:"port"` - raw string - URL string `json:"url,omitempty" csv:"url"` - Input string `json:"input,omitempty" csv:"input"` - Location string `json:"location,omitempty" csv:"location"` - Title string `json:"title,omitempty" csv:"title"` - str string - Scheme string `json:"scheme,omitempty" csv:"scheme"` - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"body,omitempty" csv:"body"` - ContentType string `json:"content_type,omitempty" csv:"content_type"` - Method string `json:"method,omitempty" csv:"method"` - Host string `json:"host,omitempty" csv:"host"` - Path string `json:"path,omitempty" csv:"path"` - FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` - FinalURL string `json:"final_url,omitempty" csv:"final_url"` - ResponseHeader map[string]string `json:"header,omitempty" csv:"header"` - RawHeader string `json:"raw_header,omitempty" csv:"raw_header"` - Request string `json:"request,omitempty" csv:"request"` - ResponseTime string `json:"time,omitempty" csv:"time"` - Jarm string `json:"jarm,omitempty" csv:"jarm"` - ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` - A []string `json:"a,omitempty" csv:"a"` - CNAMEs []string `json:"cname,omitempty" csv:"cname"` - Technologies []string `json:"tech,omitempty" csv:"tech"` - Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - Words int `json:"words" csv:"words"` - Lines int `json:"lines" csv:"lines"` - StatusCode int `json:"status_code,omitempty" csv:"status_code"` - ContentLength int `json:"content_length,omitempty" csv:"content_length"` - Failed bool `json:"failed" csv:"failed"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` -} - // JSON the result func (r Result) JSON(scanopts *scanOptions) string { //nolint if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave { diff --git a/runner/types.go b/runner/types.go new file mode 100644 index 0000000..19f8256 --- /dev/null +++ b/runner/types.go @@ -0,0 +1,95 @@ +package runner + +import ( + "fmt" + "reflect" + "strings" + "time" + + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" +) + +type AsnResponse struct { + AsNumber string `json:"as-number" csv:"as-number"` + AsName string `json:"as-name" csv:"as-name"` + AsCountry string `json:"as-country" csv:"as-country"` + AsRange string `json:"as-range" csv:"as-range"` +} + +func (o AsnResponse) String() string { + return fmt.Sprintf("%v, %v, %v, %v", o.AsNumber, o.AsName, o.AsCountry, o.AsRange) +} + +// Result of a scan +type Result struct { + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" dsl:"timestamp"` + ASN interface{} `json:"asn,omitempty" csv:"asn"` + err error + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp" dsl:"csp"` + TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` + Hashes map[string]string `json:"hash,omitempty" csv:"hash" dsl:"hash,header_md5,header_mmh3,header_sha256,header_simhash,body_md5,body_mmh3,body_sha256,body_simhash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` + Port string `json:"port,omitempty" csv:"port" dsl:"port"` + raw string + URL string `json:"url,omitempty" csv:"url" dsl:"url"` + Input string `json:"input,omitempty" csv:"input" dsl:"input"` + Location string `json:"location,omitempty" csv:"location" dsl:"location"` + Title string `json:"title,omitempty" csv:"title"` + str string + Scheme string `json:"scheme,omitempty" csv:"scheme" dsl:"scheme"` + Error string `json:"error,omitempty" csv:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver" dsl:"webserver"` + ResponseBody string `json:"body,omitempty" csv:"body" dsl:"body"` + ContentType string `json:"content_type,omitempty" csv:"content_type" dsl:"content_type"` + Method string `json:"method,omitempty" csv:"method" dsl:"method"` + Host string `json:"host,omitempty" csv:"host" dsl:"host"` + Path string `json:"path,omitempty" csv:"path" dsl:"path"` + FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` + FinalURL string `json:"final_url,omitempty" csv:"final_url"` + ResponseHeader map[string]string `json:"header,omitempty" csv:"header" dsl:"header"` + RawHeader string `json:"raw_header,omitempty" csv:"raw_header"` + Request string `json:"request,omitempty" csv:"request" dsl:"request"` + ResponseTime string `json:"time,omitempty" csv:"time" dsl:"time"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` + ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` + A []string `json:"a,omitempty" csv:"a" dsl:"a"` + CNAMEs []string `json:"cname,omitempty" csv:"cname"` + Technologies []string `json:"tech,omitempty" csv:"tech"` + Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` + Words int `json:"words" csv:"words" dsl:"words"` + Lines int `json:"lines" csv:"lines" dsl:"lines"` + StatusCode int `json:"status_code,omitempty" csv:"status_code" dsl:"status_code"` + ContentLength int `json:"content_length,omitempty" csv:"content_length" dsl:"content_length"` + Failed bool `json:"failed" csv:"failed" dsl:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` +} + +// function to get dsl variables from result struct +func dslVariables() []string { + dslVarList := []string{} + t := reflect.TypeOf(Result{}) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if field.Tag.Get("dsl") != "" { + dslVarList = append(dslVarList, strings.Split(field.Tag.Get("dsl"), ",")...) + } + + if field.Name == "CSPData" { + u := reflect.TypeOf(httpx.CSPData{}) + for j := 0; j < u.NumField(); j++ { + field := u.Field(j) + if field.Tag.Get("dsl") != "" { + dslVarList = append(dslVarList, strings.Split(field.Tag.Get("dsl"), ",")...) + } + } + } + } + return dslVarList +} From d48afd2a79fbef63640fa16d39c9d68c639a2136 Mon Sep 17 00:00:00 2001 From: Ismael Arias Date: Wed, 21 Sep 2022 16:33:36 +0200 Subject: [PATCH 256/355] Added flag for single or comma separated stdin input values (#737) (#759) * Added flag for single or comma separated stdin input values (#737) * Replaced the logic in the parsing of input target hosts flag. Using the goflags' StringSlice now. Co-authored-by: Sandeep Singh --- runner/options.go | 2 ++ runner/runner.go | 51 +++++++++++++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/runner/options.go b/runner/options.go index afb7138..ddd209f 100644 --- a/runner/options.go +++ b/runner/options.go @@ -142,6 +142,7 @@ type Options struct { HTTPProxy string SocksProxy string InputFile string + InputTargetHost goflags.StringSlice Methods string RequestURI string RequestURIs string @@ -260,6 +261,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("input", "Input", flagSet.StringVarP(&options.InputFile, "list", "l", "", "input file containing list of hosts to process"), flagSet.StringVarP(&options.InputRawRequest, "request", "rr", "", "file containing raw request"), + flagSet.StringSliceVarP(&options.InputTargetHost, "target", "u", nil, "input target host(s) to probe", goflags.CommaSeparatedStringSliceOptions), ) flagSet.CreateGroup("Probes", "Probes", diff --git a/runner/runner.go b/runner/runner.go index 7d90302..f1fdc8a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -320,8 +320,18 @@ func (r *Runner) prepareInputPaths() { } func (r *Runner) prepareInput() { - // check if file has been provided var numHosts int + // check if input target host(s) have been provided + if len(r.options.InputTargetHost) > 0 { + for _, target := range r.options.InputTargetHost { + expandedTarget := r.countTargetFromRawTarget(target) + if expandedTarget > 0 { + numHosts += expandedTarget + r.hm.Set(target, nil) //nolint + } + } + } + // check if file has been provided if fileutil.FileExists(r.options.InputFile) { finput, err := os.Open(r.options.InputFile) if err != nil { @@ -445,30 +455,33 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { for scanner.Scan() { target := strings.TrimSpace(scanner.Text()) // Used just to get the exact number of targets - if target == "" { - continue - } - if _, ok := r.hm.Get(target); ok { - continue - } - - // if the target is ip or host it counts as 1 - expandedTarget := 1 - // input can be a cidr - if iputil.IsCIDR(target) { - // so we need to count the ips - if ipsCount, err := mapcidr.AddressCount(target); err == nil && ipsCount > 0 { - expandedTarget = int(ipsCount) - } + expandedTarget := r.countTargetFromRawTarget(target) + if expandedTarget > 0 { + numTargets += expandedTarget + r.hm.Set(target, nil) //nolint } - - numTargets += expandedTarget - r.hm.Set(target, nil) //nolint } err = finput.Close() return numTargets, err } +func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int) { + if rawTarget == "" { + return 0 + } + if _, ok := r.hm.Get(rawTarget); ok { + return 0 + } + + expandedTarget := 1 + if iputil.IsCIDR(rawTarget) { + if ipsCount, err := mapcidr.AddressCount(rawTarget); err == nil && ipsCount > 0 { + expandedTarget = int(ipsCount) + } + } + return expandedTarget +} + var ( lastRequestsCount float64 ) From 5a7f616418473c33e00873a5866a19d72289105c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 10:27:12 +0000 Subject: [PATCH 257/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.60 to 0.0.61. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.60...v0.0.61) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6c6543d..d7cae1f 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.60 + github.com/projectdiscovery/wappalyzergo v0.0.61 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index ab89102..86854e8 100644 --- a/go.sum +++ b/go.sum @@ -327,8 +327,8 @@ github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgt github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.60 h1:xD1l+LrW3LLY7OlsKYtizEwfv7gnttF/w8at8ZHJ2s0= -github.com/projectdiscovery/wappalyzergo v0.0.60/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.61 h1:c7azmo5tEWqRUzq60Jfevqdjhn6FHEdaq/UARIHKIeY= +github.com/projectdiscovery/wappalyzergo v0.0.61/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= From 42776e282df82efd130fb2939c58f99a037f4b5c Mon Sep 17 00:00:00 2001 From: Gwendal Le Coguic Date: Fri, 30 Sep 2022 21:33:30 +0200 Subject: [PATCH 258/355] extractRegexps -er on body and header (resp.Raw) (#787) Co-authored-by: Sandeep Singh --- runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index f1fdc8a..a32678a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1333,7 +1333,7 @@ retry: var extractResult = map[string][]string{} if scanopts.extractRegexps != nil { for regex, compiledRegex := range scanopts.extractRegexps { - matches := compiledRegex.FindAllString(string(resp.Data), -1) + matches := compiledRegex.FindAllString(string(resp.Raw), -1) if len(matches) > 0 { matches = sliceutil.Dedupe(matches) builder.WriteString(" [" + strings.Join(matches, ",") + "]") From 6e93761f429af05bc2c126f8a340a4cb65b06a17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Oct 2022 19:49:17 +0530 Subject: [PATCH 259/355] chore(deps): bump github.com/projectdiscovery/fastdialer from 0.0.16 to 0.0.17 (#794) chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.16 to 0.0.17. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.16...v0.0.17) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d7cae1f..9ed3547 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect - github.com/projectdiscovery/fastdialer v0.0.16 + github.com/projectdiscovery/fastdialer v0.0.17 github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c diff --git a/go.sum b/go.sum index 86854e8..a5f7d0b 100644 --- a/go.sum +++ b/go.sum @@ -267,8 +267,8 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.16 h1:0FcWVAtm6ftVALQpckdlKTeZcJi277HduvGXsBDO02Q= -github.com/projectdiscovery/fastdialer v0.0.16/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= +github.com/projectdiscovery/fastdialer v0.0.17 h1:wT7LinZAcyEzkfhJLhWNQrA9m79+hTd6CJ7aWTJfXjk= +github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= From 04b3ea3e2bd63dc592f931603a02a0ab0929145d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Oct 2022 19:49:34 +0530 Subject: [PATCH 260/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.61 to 0.0.62 (#790) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.61 to 0.0.62. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.61...v0.0.62) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9ed3547..0d60b11 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.61 + github.com/projectdiscovery/wappalyzergo v0.0.62 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index a5f7d0b..0528088 100644 --- a/go.sum +++ b/go.sum @@ -327,8 +327,8 @@ github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgt github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.61 h1:c7azmo5tEWqRUzq60Jfevqdjhn6FHEdaq/UARIHKIeY= -github.com/projectdiscovery/wappalyzergo v0.0.61/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.62 h1:bpGTH6IpRsjl+fovGTnmNxl5wmcvvU4pI+ePupBvVjM= +github.com/projectdiscovery/wappalyzergo v0.0.62/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= From d0fe1ab21397af4e038102445797b7584a2cc8b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Oct 2022 19:49:48 +0530 Subject: [PATCH 261/355] chore(deps): bump github.com/projectdiscovery/tlsx from 0.0.7 to 0.0.8 (#795) Bumps [github.com/projectdiscovery/tlsx](https://github.com/projectdiscovery/tlsx) from 0.0.7 to 0.0.8. - [Release notes](https://github.com/projectdiscovery/tlsx/releases) - [Changelog](https://github.com/projectdiscovery/tlsx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/tlsx/compare/v0.0.7...v0.0.8) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/tlsx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 21 +++++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 0d60b11..24d8c72 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/fastdialer v0.0.17 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 + github.com/projectdiscovery/fileutil v0.0.1 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.0 github.com/projectdiscovery/gologger v1.1.4 @@ -38,7 +38,7 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20220909164309-bea034e7d591 + golang.org/x/net v0.0.0-20220927171203-f486391704dc golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect golang.org/x/text v0.3.7 ) @@ -50,7 +50,7 @@ require ( github.com/ammario/ipisp/v2 v2.0.0 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 - github.com/projectdiscovery/tlsx v0.0.7 + github.com/projectdiscovery/tlsx v0.0.8 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e ) @@ -75,7 +75,7 @@ require ( github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/gogo/protobuf v1.3.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index 0528088..c197c85 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,9 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -181,6 +182,7 @@ github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/l github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -274,8 +276,9 @@ github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIre github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= -github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 h1:4o97N9ftX1J3iKlIRVMPVOVZs4qbCczJvoFF2WA40t4= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= +github.com/projectdiscovery/fileutil v0.0.1 h1:3K3UqCDOan3LsvWhV0nyvVuMWSwCloNPUJIGcXsi1os= +github.com/projectdiscovery/fileutil v0.0.1/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2LYinWCSEmzZOQsJCLg= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= @@ -323,8 +326,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ= github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM= -github.com/projectdiscovery/tlsx v0.0.7 h1:McoDo4Ju7aetogatU4lVTRcQpxkf0qgQSoWgtkDavCk= -github.com/projectdiscovery/tlsx v0.0.7/go.mod h1:/ZCk/zzyuDdXx2E511yhNIj3LERnGUXghqkASBbdh5M= +github.com/projectdiscovery/tlsx v0.0.8 h1:+YsPi1sQ6UZUPA6eA8t1RRdbuNzy2cFb9DvvlJFCq9o= +github.com/projectdiscovery/tlsx v0.0.8/go.mod h1:TlIEi84Vfhs5MVGbb41UAiLkDcSIF8feca6XhyNU2KY= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= github.com/projectdiscovery/wappalyzergo v0.0.62 h1:bpGTH6IpRsjl+fovGTnmNxl5wmcvvU4pI+ePupBvVjM= @@ -398,6 +401,7 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/ github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -442,6 +446,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= @@ -459,6 +464,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -475,13 +481,14 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ= +golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -545,7 +552,9 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= From e31c1250a2ed89506e6b67071b078d7bc7c95d11 Mon Sep 17 00:00:00 2001 From: Sajad Date: Thu, 6 Oct 2022 17:11:12 +0530 Subject: [PATCH 262/355] add dsl match/filter condtion (#782) * add dsl match/filter condtion * update README * add test cases --- README.md | 2 ++ cmd/integration-test/http.go | 52 +++++++++++++++++++++++++++++++ go.mod | 3 ++ go.sum | 5 +++ internal/testutils/integration.go | 4 +-- runner/options.go | 4 +++ runner/runner.go | 30 ++++++++++++++++++ runner/types.go | 17 ++++++++++ 8 files changed, 115 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ada6ee..7c657e9 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ PROBES: -probe display probe status MATCHERS: + -mdc -match-condition string match response with dsl expression condition -mc, -match-code string match response with specified status code (-mc 200,302) -ml, -match-length string match response with specified content length (-ml 100,102) -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) @@ -124,6 +125,7 @@ EXTRACTOR: -ep, -extract-preset string[] Display response content with matched preset regex FILTERS: + -fdc -filter-condition string filter response with dsl expression condition -fc, -filter-code string filter response with specified status code (-fc 403,401) -fl, -filter-length string filter response with specified content length (-fl 23,33) -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 4951995..5637961 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -28,6 +28,8 @@ var httpTestcases = map[string]testutils.TestCase{ "Standard HTTP GET Request with match response time": &standardHttpGet{mrt: true, inputValue: "\"<10s\""}, "Standard HTTP GET Request with filter response time": &standardHttpGet{frt: true, inputValue: "\">3ms\""}, "Multiple Custom Header": &customHeader{inputData: []string{"-debug-req", "-H", "'user-agent: test'", "-H", "'foo: bar'"}, expectedOutput: []string{"User-Agent: test", "Foo: bar"}}, + "Output Match Condition": &outputMatchCondition{inputData: []string{"-silent", "-mdc", "\"status_code == 200\""}}, + "Output Filter Condition": &outputFilterCondition{inputData: []string{"-silent", "-fdc", "\"status_code == 400\""}}, } type standardHttpGet struct { @@ -325,3 +327,53 @@ func (h *customHeader) Execute() error { } return nil } + +type outputMatchCondition struct { + inputData []string +} + +func (h *outputMatchCondition) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(200) + fmt.Fprint(w, `{"status": "ok"}`) + })) + ts = httptest.NewServer(router) + defer ts.Close() + results, err := testutils.RunHttpxAndGetResults(ts.URL, false, h.inputData...) + if err != nil { + return err + } + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} + +type outputFilterCondition struct { + inputData []string +} + +func (h *outputFilterCondition) Execute() error { + var ts *httptest.Server + router := httprouter.New() + router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(200) + fmt.Fprint(w, `{"status": "ok"}`) + })) + ts = httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunHttpxAndGetResults(ts.URL, false, h.inputData...) + if err != nil { + return err + } + + if len(results) != 1 { + return errIncorrectResultsCount(results) + } + return nil +} diff --git a/go.mod b/go.mod index 24d8c72..6ca103f 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,8 @@ require ( github.com/ammario/ipisp/v2 v2.0.0 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 + github.com/mitchellh/mapstructure v1.1.2 + github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/tlsx v0.0.8 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e @@ -58,6 +60,7 @@ require ( require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/DataDog/zstd v1.4.5 // indirect + github.com/Knetic/govaluate v3.0.0+incompatible // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect diff --git a/go.sum b/go.sum index c197c85..17c7552 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= +github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= +github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -223,6 +225,7 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -269,6 +272,8 @@ github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKS github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= +github.com/projectdiscovery/dsl v0.0.3 h1:oWlZZaSADqoyfJdUHWqAzpB65NpvLukZQGFv1uTtU3g= +github.com/projectdiscovery/dsl v0.0.3/go.mod h1:ST66slxtp7fsFqTOq3k6+1TDV7RH1Moz3sPBqUGn4Fg= github.com/projectdiscovery/fastdialer v0.0.17 h1:wT7LinZAcyEzkfhJLhWNQrA9m79+hTd6CJ7aWTJfXjk= github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go index 8b5af02..e5c23e4 100644 --- a/internal/testutils/integration.go +++ b/internal/testutils/integration.go @@ -1,10 +1,10 @@ package testutils import ( + "fmt" "os" "os/exec" "strings" - "fmt" ) // RunNucleiAndGetResults returns a list of results for a template @@ -76,7 +76,7 @@ func RunHttpxBinaryAndGetResults(target string, httpxBinary string, debug bool, parts = append(parts, i) } } - return parts,nil + return parts, nil } // TestCase is a single integration test case diff --git a/runner/options.go b/runner/options.go index ddd209f..ceee9c9 100644 --- a/runner/options.go +++ b/runner/options.go @@ -249,6 +249,8 @@ type Options struct { OutputFilterResponseTime string HealthCheck bool ListDSLVariable bool + OutputFilterCondition string + OutputMatchCondition string } // ParseOptions parses the command line options for application @@ -297,6 +299,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), flagSet.StringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", nil, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"), + flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl expression condition"), ) flagSet.CreateGroup("extractor", "Extractor", @@ -314,6 +317,7 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), flagSet.StringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", nil, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders), goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), + flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl expression condition"), ) flagSet.CreateGroup("rate-limit", "Rate-Limit", diff --git a/runner/runner.go b/runner/runner.go index a32678a..d154f78 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -24,6 +24,7 @@ import ( "golang.org/x/exp/maps" + dsl "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" @@ -606,6 +607,35 @@ func (r *Runner) RunEnumeration() { } // apply matchers and filters + if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" { + respObj := make(map[string]interface{}) + if err := decodeResponse(resp, respObj); err != nil { + gologger.Warning().Msgf("Could not decode response: %s\n", err) + } else { + if r.options.OutputMatchCondition != "" { + res, err := dsl.EvalExpr(r.options.OutputMatchCondition, respObj) + if err != nil { + gologger.Error().Msgf("Could not evaluate match condition: %s\n", err) + + } else { + if res == false { + continue + } + } + } + if r.options.OutputFilterCondition != "" { + res, err := dsl.EvalExpr(r.options.OutputFilterCondition, respObj) + if err != nil { + gologger.Error().Msgf("Could not evaluate filter condition: %s\n", err) + } else { + if res == true { + continue + } + } + } + } + } + if len(r.options.filterStatusCode) > 0 && slice.IntSliceContains(r.options.filterStatusCode, resp.StatusCode) { continue } diff --git a/runner/types.go b/runner/types.go index 19f8256..4fdbb07 100644 --- a/runner/types.go +++ b/runner/types.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/mitchellh/mapstructure" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" ) @@ -93,3 +94,19 @@ func dslVariables() []string { } return dslVarList } + +func decodeResponse(resp Result, respObj interface{}) error { + config := &mapstructure.DecoderConfig{ + TagName: "dsl", + Result: &respObj, + } + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return fmt.Errorf("error creating decoder: %v", err) + } + err = decoder.Decode(resp) + if err != nil { + return fmt.Errorf("error decoding: %v", err) + } + return nil +} From 76cb70f796e91ae27ce764244d14876605ca3784 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 6 Oct 2022 13:00:39 -0500 Subject: [PATCH 263/355] replaced go.uber.org/ratelimit with projectdiscovery/ratelimit --- go.mod | 3 +-- go.sum | 6 ++---- runner/runner.go | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 6ca103f..b9beb20 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,6 @@ require ( github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect - go.uber.org/ratelimit v0.2.0 golang.org/x/net v0.0.0-20220927171203-f486391704dc golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect golang.org/x/text v0.3.7 @@ -52,6 +51,7 @@ require ( github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/mitchellh/mapstructure v1.1.2 github.com/projectdiscovery/dsl v0.0.3 + github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa github.com/projectdiscovery/tlsx v0.0.8 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e @@ -62,7 +62,6 @@ require ( github.com/DataDog/zstd v1.4.5 // indirect github.com/Knetic/govaluate v3.0.0+incompatible // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect - github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 17c7552..c961d32 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,6 @@ github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/ammario/ipisp/v2 v2.0.0 h1:/aRMp5srZViiBfOUGzl/Esqae4s0MDDzm9buhGcZ0XU= github.com/ammario/ipisp/v2 v2.0.0/go.mod h1:bQ6KAL5LnYYEj6olUn+Bzv/im/4Esa5oGkbv9b+uOjo= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -310,6 +308,8 @@ github.com/projectdiscovery/mapcidr v1.0.2/go.mod h1:9OuUDjA+FilmXToex0xMfQb0os6 github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= +github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa h1:XTRcow+zyZUYxOlLsSp1AdM4Jl+i/OpPbOZPsfU/IAI= +github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= @@ -424,8 +424,6 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= -go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/runner/runner.go b/runner/runner.go index d154f78..02f3fce 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -43,7 +43,7 @@ import ( "github.com/projectdiscovery/urlutil" "github.com/remeh/sizedwaitgroup" - "go.uber.org/ratelimit" + "github.com/projectdiscovery/ratelimit" // automatic fd max increase if running as root _ "github.com/projectdiscovery/fdmax/autofdmax" From 0c277240a363d6d87792546b83ce7124f3a20a1e Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 6 Oct 2022 13:31:39 -0500 Subject: [PATCH 264/355] updates related to projectdiscovery/ratelimit --- runner/runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 02f3fce..e8ab778 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -292,11 +292,11 @@ func New(options *Options) (*Runner, error) { runner.hm = hm if options.RateLimitMinute > 0 { - runner.ratelimiter = ratelimit.New(options.RateLimitMinute, ratelimit.Per(60*time.Second)) + runner.ratelimiter = *ratelimit.New(context.Background(), options.RateLimitMinute, time.Minute) } else if options.RateLimit > 0 { - runner.ratelimiter = ratelimit.New(options.RateLimit) + runner.ratelimiter = *ratelimit.New(context.Background(), options.RateLimit, time.Second) } else { - runner.ratelimiter = ratelimit.NewUnlimited() + runner.ratelimiter = *ratelimit.NewUnlimited(context.Background()) } if options.HostMaxErrors >= 0 { From a457828bb072a63c5712727f54130c2bd651819b Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Thu, 6 Oct 2022 21:04:24 -0500 Subject: [PATCH 265/355] added GOPRIVATE in workflow --- .github/workflows/build-test.yml | 2 ++ .github/workflows/functional-test.yml | 2 ++ .github/workflows/lint-test.yml | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 74ccc30..4c413cf 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -24,6 +24,8 @@ jobs: - name: Build run: go build . working-directory: cmd/httpx/ + env: + GOPRIVATE: "github.com/projectdiscovery" - name: Test run: go test ./... diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 841226c..c0f843c 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -25,3 +25,5 @@ jobs: chmod +x run.sh bash run.sh ${{ matrix.os }} working-directory: cmd/functional-test + env: + GOPRIVATE: "github.com/projectdiscovery" diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index d8b1dbc..577f8b9 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -20,4 +20,6 @@ jobs: with: version: latest args: --timeout 5m - working-directory: . \ No newline at end of file + working-directory: . + env: + GOPRIVATE: "github.com/projectdiscovery" From f69a34ee724d6a2628303a6261166e5b67db9c0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:21:08 +0000 Subject: [PATCH 266/355] chore(deps): bump github.com/mitchellh/mapstructure from 1.1.2 to 1.5.0 Bumps [github.com/mitchellh/mapstructure](https://github.com/mitchellh/mapstructure) from 1.1.2 to 1.5.0. - [Release notes](https://github.com/mitchellh/mapstructure/releases) - [Changelog](https://github.com/mitchellh/mapstructure/blob/main/CHANGELOG.md) - [Commits](https://github.com/mitchellh/mapstructure/compare/v1.1.2...v1.5.0) --- updated-dependencies: - dependency-name: github.com/mitchellh/mapstructure dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 6ca103f..e55dd4d 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/ammario/ipisp/v2 v2.0.0 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 - github.com/mitchellh/mapstructure v1.1.2 + github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/tlsx v0.0.8 go.uber.org/multierr v1.8.0 diff --git a/go.sum b/go.sum index 17c7552..ef9d6c9 100644 --- a/go.sum +++ b/go.sum @@ -225,8 +225,9 @@ github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From 5005027138128b6f1c55ffc6b2b3f7f2fefef9c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 10:04:48 +0000 Subject: [PATCH 267/355] chore(deps): bump golang from 1.19.1-alpine to 1.19.2-alpine Bumps golang from 1.19.1-alpine to 1.19.2-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e596015..4c0ffbb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.1-alpine AS builder +FROM golang:1.19.2-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 60640d5bb7fb032a95fc6ab6cb8d9be815fc8be8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 10:18:25 +0000 Subject: [PATCH 268/355] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.0 to 0.1.1. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.0...v0.1.1) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e55dd4d..15c4195 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.1 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.1.0 + github.com/projectdiscovery/goflags v0.1.1 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 diff --git a/go.sum b/go.sum index ef9d6c9..ee580ed 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/projectdiscovery/fileutil v0.0.1/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2L github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= -github.com/projectdiscovery/goflags v0.1.0 h1:Z7sUVK8wgH6aGJWinmGQEtsn+GNf/0RQ+z1wQcpCeeA= -github.com/projectdiscovery/goflags v0.1.0/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= +github.com/projectdiscovery/goflags v0.1.1 h1:AEtT14D9OC10HWyZwDQaSLjuK8ZKoBrSYlsLItvMKZI= +github.com/projectdiscovery/goflags v0.1.1/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= From e9b7fe7350055a2e25da184a0317b83a84262938 Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Mon, 10 Oct 2022 17:02:36 -0500 Subject: [PATCH 269/355] removed goprivate env --- .github/workflows/build-test.yml | 4 +--- .github/workflows/functional-test.yml | 2 -- .github/workflows/lint-test.yml | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 4c413cf..afa6784 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -24,9 +24,7 @@ jobs: - name: Build run: go build . working-directory: cmd/httpx/ - env: - GOPRIVATE: "github.com/projectdiscovery" - + - name: Test run: go test ./... working-directory: . diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index c0f843c..841226c 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -25,5 +25,3 @@ jobs: chmod +x run.sh bash run.sh ${{ matrix.os }} working-directory: cmd/functional-test - env: - GOPRIVATE: "github.com/projectdiscovery" diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 577f8b9..98ecbd6 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -21,5 +21,3 @@ jobs: version: latest args: --timeout 5m working-directory: . - env: - GOPRIVATE: "github.com/projectdiscovery" From 1db89a81383593d5504c2a2c989af12309236819 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 16:01:05 +0530 Subject: [PATCH 270/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.62 to 0.0.64 (#804) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.62 to 0.0.64. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.62...v0.0.64) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e55dd4d..111196d 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.62 + github.com/projectdiscovery/wappalyzergo v0.0.64 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index ef9d6c9..53e6e53 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/projectdiscovery/tlsx v0.0.8 h1:+YsPi1sQ6UZUPA6eA8t1RRdbuNzy2cFb9Dvvl github.com/projectdiscovery/tlsx v0.0.8/go.mod h1:TlIEi84Vfhs5MVGbb41UAiLkDcSIF8feca6XhyNU2KY= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.62 h1:bpGTH6IpRsjl+fovGTnmNxl5wmcvvU4pI+ePupBvVjM= -github.com/projectdiscovery/wappalyzergo v0.0.62/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.64 h1:OXjTUDfbJJqM9bTT/V6lOiv29uaAxrkARgiZqdio6Kg= +github.com/projectdiscovery/wappalyzergo v0.0.64/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= From f396eefccbc731d4b323422a66e9d06a6f1ae53a Mon Sep 17 00:00:00 2001 From: LuitelSamikshya Date: Tue, 11 Oct 2022 15:02:17 -0500 Subject: [PATCH 271/355] spacing in build workflow --- .github/workflows/build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index afa6784..74ccc30 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -24,7 +24,7 @@ jobs: - name: Build run: go build . working-directory: cmd/httpx/ - + - name: Test run: go test ./... working-directory: . From 7f98e13c4a135baeeeffb8c58fee66ef14439c39 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 12 Oct 2022 00:15:04 +0200 Subject: [PATCH 272/355] updating goflags syntax --- runner/healthcheck.go | 4 ++-- runner/options.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/healthcheck.go b/runner/healthcheck.go index c0192f9..ddb3937 100644 --- a/runner/healthcheck.go +++ b/runner/healthcheck.go @@ -10,9 +10,9 @@ import ( "github.com/projectdiscovery/goflags" ) -func DoHealthCheck(options *Options) string { +func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { // RW permissions on config file - cfgFilePath, _ := goflags.GetConfigFilePath() + cfgFilePath, _ := flagSet.GetConfigFilePath() var test strings.Builder test.WriteString(fmt.Sprintf("Version: %s\n", Version)) test.WriteString(fmt.Sprintf("Operative System: %s\n", runtime.GOOS)) diff --git a/runner/options.go b/runner/options.go index ceee9c9..488b4b1 100644 --- a/runner/options.go +++ b/runner/options.go @@ -399,7 +399,7 @@ func ParseOptions() *Options { _ = flagSet.Parse() if options.HealthCheck { - gologger.Print().Msgf("%s\n", DoHealthCheck(options)) + gologger.Print().Msgf("%s\n", DoHealthCheck(options, flagSet)) os.Exit(0) } From 0dac46522dd6c189fbb6d7f69180fb1619738652 Mon Sep 17 00:00:00 2001 From: Shubham S Jagtap <63872951+ShubhamJagtap2000@users.noreply.github.com> Date: Wed, 12 Oct 2022 12:36:19 +0530 Subject: [PATCH 273/355] [docs] Update README.md (#808) Update README.md Co-authored-by: Sandeep Singh --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c657e9..3c1b621 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Installation • Usage • Running httpx • - Notes • + Notes • Join Discord

From cbf6f3e8c00ed831ef80e5e5e274350edaab35d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:09:35 +0000 Subject: [PATCH 274/355] chore(deps): bump github.com/projectdiscovery/mapcidr Bumps [github.com/projectdiscovery/mapcidr](https://github.com/projectdiscovery/mapcidr) from 1.0.2 to 1.0.3. - [Release notes](https://github.com/projectdiscovery/mapcidr/releases) - [Changelog](https://github.com/projectdiscovery/mapcidr/blob/master/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/mapcidr/compare/v1.0.2...v1.0.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/mapcidr dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 3b29800..9f7330a 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 - github.com/projectdiscovery/mapcidr v1.0.2 + github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 @@ -54,7 +54,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa github.com/projectdiscovery/tlsx v0.0.8 go.uber.org/multierr v1.8.0 - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e + golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 ) require ( diff --git a/go.sum b/go.sum index 4e4cc0a..45679d7 100644 --- a/go.sum +++ b/go.sum @@ -304,8 +304,8 @@ github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1: github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= -github.com/projectdiscovery/mapcidr v1.0.2 h1:ewn7hoZz8P1TJ4VqG//XMlQKg5zhj2hnEx3C9jf3sQA= -github.com/projectdiscovery/mapcidr v1.0.2/go.mod h1:9OuUDjA+FilmXToex0xMfQb0os6qMaoiuZCksai0asY= +github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= +github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= @@ -440,8 +440,8 @@ golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 h1:17k44ji3KFYG94XS5QEFC8pyuOlMh3IoR+vkmTZmJJs= +golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From d46e1239ec750d9ed3d132c7b62c2b23110a6357 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 10:11:41 +0000 Subject: [PATCH 275/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.64 to 0.0.65. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.64...v0.0.65) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3b29800..690af34 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.64 + github.com/projectdiscovery/wappalyzergo v0.0.65 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 4e4cc0a..0f2fe95 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/projectdiscovery/tlsx v0.0.8 h1:+YsPi1sQ6UZUPA6eA8t1RRdbuNzy2cFb9Dvvl github.com/projectdiscovery/tlsx v0.0.8/go.mod h1:TlIEi84Vfhs5MVGbb41UAiLkDcSIF8feca6XhyNU2KY= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.64 h1:OXjTUDfbJJqM9bTT/V6lOiv29uaAxrkARgiZqdio6Kg= -github.com/projectdiscovery/wappalyzergo v0.0.64/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.65 h1:M1/r75lJI5rezDmfrARQiNNm+Ii9o3jsV+Q+F69Lcso= +github.com/projectdiscovery/wappalyzergo v0.0.65/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= From 4bb6350dc7b51e89104fd25f47e1567099e56469 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Oct 2022 10:10:10 +0000 Subject: [PATCH 276/355] chore(deps): bump golang.org/x/text from 0.3.7 to 0.4.0 Bumps [golang.org/x/text](https://github.com/golang/text) from 0.3.7 to 0.4.0. - [Release notes](https://github.com/golang/text/releases) - [Commits](https://github.com/golang/text/compare/v0.3.7...v0.4.0) --- updated-dependencies: - dependency-name: golang.org/x/text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3b29800..83f679d 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect golang.org/x/net v0.0.0-20220927171203-f486391704dc golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect - golang.org/x/text v0.3.7 + golang.org/x/text v0.4.0 ) require github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index 4e4cc0a..104f92b 100644 --- a/go.sum +++ b/go.sum @@ -542,8 +542,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 12fee4f3eefda88727f96c3d030ba256428ac7f0 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Thu, 20 Oct 2022 02:48:01 -0500 Subject: [PATCH 277/355] include dsl tag for all json keys (#812) * include dsl tag for all json keys * removed dsl tag and built list from json tag * asn interface changed * refactoring expression engine - Using mapstructure for uniform mapping - Flat key:value in expressions/variables list * adding fields to ignore Co-authored-by: Mzack9999 --- README.md | 2 +- common/httpx/csp.go | 2 +- go.mod | 13 ++-- go.sum | 23 ++++--- runner/options.go | 6 +- runner/runner.go | 62 ++++++++++--------- runner/types.go | 144 ++++++++++++++++++++++---------------------- 7 files changed, 136 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index 3c1b621..dc7ba2c 100644 --- a/README.md +++ b/README.md @@ -486,7 +486,7 @@ func main() { httpxRunner, err := runner.New(&options) if err != nil { - log.Fatal() + log.Fatal(err) } defer httpxRunner.Close() diff --git a/common/httpx/csp.go b/common/httpx/csp.go index d663baf..e1c4831 100644 --- a/common/httpx/csp.go +++ b/common/httpx/csp.go @@ -19,7 +19,7 @@ var CSPHeaders = []string{ // CSPData contains the Content-Security-Policy domain list type CSPData struct { - Domains []string `json:"domains,omitempty" dsl:"domains"` + Domains []string `json:"domains,omitempty"` } // CSPGrab fills the CSPData diff --git a/go.mod b/go.mod index b3faef3..0a97d08 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/microcosm-cc/bluemonday v1.0.20 + github.com/microcosm-cc/bluemonday v1.0.21 github.com/miekg/dns v1.1.50 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 @@ -37,8 +37,8 @@ require ( github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect - golang.org/x/net v0.0.0-20220927171203-f486391704dc - golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect + golang.org/x/net v0.1.0 + golang.org/x/sys v0.1.0 // indirect golang.org/x/text v0.4.0 ) @@ -47,12 +47,14 @@ require github.com/spaolacci/murmur3 v1.1.0 require ( github.com/PuerkitoBio/goquery v1.8.0 github.com/ammario/ipisp/v2 v2.0.0 + github.com/bxcodec/faker/v4 v4.0.0-beta.3 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/dsl v0.0.3 + github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655 github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa - github.com/projectdiscovery/tlsx v0.0.8 + github.com/projectdiscovery/tlsx v0.0.9 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 ) @@ -97,8 +99,9 @@ require ( github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect + github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c // indirect - golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect + golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/protobuf v1.27.1 // indirect diff --git a/go.sum b/go.sum index 62edc5d..51c8345 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd3 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= +github.com/bxcodec/faker/v4 v4.0.0-beta.3 h1:gqYNBvN72QtzKkYohNDKQlm+pg+uwBDVMN28nWHS18k= +github.com/bxcodec/faker/v4 v4.0.0-beta.3/go.mod h1:m6+Ch1Lj3fqW/unZmvkXIdxWS5+XQWPWxcbbQW2X+Ho= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -217,8 +219,9 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/p github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= -github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= +github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= +github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= @@ -306,6 +309,8 @@ github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJ github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= +github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655 h1:6FTqSAohPHeBnzTypby8Gk88zv03PfrRMKQtRCFELrU= +github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655/go.mod h1:NIgEkt1yYqHWYi1Razavwa1J2hinHfSwD287ZtKBQjU= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= @@ -332,8 +337,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ= github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM= -github.com/projectdiscovery/tlsx v0.0.8 h1:+YsPi1sQ6UZUPA6eA8t1RRdbuNzy2cFb9DvvlJFCq9o= -github.com/projectdiscovery/tlsx v0.0.8/go.mod h1:TlIEi84Vfhs5MVGbb41UAiLkDcSIF8feca6XhyNU2KY= +github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= +github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= github.com/projectdiscovery/wappalyzergo v0.0.65 h1:M1/r75lJI5rezDmfrARQiNNm+Ii9o3jsV+Q+F69Lcso= @@ -413,6 +418,7 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 h1:17HHAgFKlLcZsDOjBOUrd5hDihb1ggf+1a5dTbkgkIY= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw= github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c h1:ufDm/IlBYZYLuiqvQuhpTKwrcAS2OlXEzWbDvTVGbSQ= @@ -435,8 +441,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= -golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 h1:vJ2V3lFLg+bBhgroYuRfyN583UzVveQmIXjc8T/y3to= +golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= @@ -485,8 +491,8 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ= -golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -533,8 +539,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/runner/options.go b/runner/options.go index 488b4b1..268d64d 100644 --- a/runner/options.go +++ b/runner/options.go @@ -415,7 +415,11 @@ func ParseOptions() *Options { gologger.Fatal().Msgf("%s\n", err) } if options.ListDSLVariable { - for _, dsl := range dslVariables() { + dslVars, err := dslVariables() + if err != nil { + gologger.Fatal().Msgf("%s\n", err) + } + for _, dsl := range dslVars { gologger.Print().Msg(dsl) } os.Exit(0) diff --git a/runner/runner.go b/runner/runner.go index e8ab778..5520788 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -28,6 +28,7 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" + "github.com/projectdiscovery/mapsutil" "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" @@ -42,8 +43,8 @@ import ( "github.com/projectdiscovery/stringsutil" "github.com/projectdiscovery/urlutil" - "github.com/remeh/sizedwaitgroup" "github.com/projectdiscovery/ratelimit" + "github.com/remeh/sizedwaitgroup" // automatic fd max increase if running as root _ "github.com/projectdiscovery/fdmax/autofdmax" @@ -608,29 +609,36 @@ func (r *Runner) RunEnumeration() { // apply matchers and filters if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" { - respObj := make(map[string]interface{}) - if err := decodeResponse(resp, respObj); err != nil { + rawMap, err := ResultToMap(resp) + if err != nil { gologger.Warning().Msgf("Could not decode response: %s\n", err) - } else { - if r.options.OutputMatchCondition != "" { - res, err := dsl.EvalExpr(r.options.OutputMatchCondition, respObj) - if err != nil { - gologger.Error().Msgf("Could not evaluate match condition: %s\n", err) - - } else { - if res == false { - continue - } + continue + } + + flatMap := make(map[string]any) + mapsutil.Walk(rawMap, func(k string, v any) { + flatMap[k] = v + }) + + if r.options.OutputMatchCondition != "" { + res, err := dsl.EvalExpr(r.options.OutputMatchCondition, flatMap) + if err != nil { + gologger.Error().Msgf("Could not evaluate match condition: %s\n", err) + continue + } else { + if res == false { + continue } } - if r.options.OutputFilterCondition != "" { - res, err := dsl.EvalExpr(r.options.OutputFilterCondition, respObj) - if err != nil { - gologger.Error().Msgf("Could not evaluate filter condition: %s\n", err) - } else { - if res == true { - continue - } + } + if r.options.OutputFilterCondition != "" { + res, err := dsl.EvalExpr(r.options.OutputFilterCondition, flatMap) + if err != nil { + gologger.Error().Msgf("Could not evaluate filter condition: %s\n", err) + continue + } else { + if res == true { + continue } } } @@ -1232,7 +1240,7 @@ retry: var serverResponseRaw string var request string var rawResponseHeader string - var responseHeader map[string]string + var responseHeader map[string]interface{} if scanopts.ResponseInStdout { serverResponseRaw = string(resp.Data) request = string(requestDump) @@ -1290,7 +1298,7 @@ retry: ip = hp.Dialer.GetDialedIP(URL.Host) } - var asnResponse interface{ String() string } + var asnResponse *AsnResponse if r.options.Asn { lookupResult, err := ipisp.LookupIP(context.Background(), net.ParseIP(ip)) if err != nil { @@ -1298,7 +1306,7 @@ retry: } if lookupResult != nil { lookupResult.ISPName = stringsutil.TrimSuffixAny(strings.ReplaceAll(lookupResult.ISPName, lookupResult.Country, ""), ", ", " ") - asnResponse = AsnResponse{ + asnResponse = &AsnResponse{ AsNumber: lookupResult.ASN.String(), AsName: lookupResult.ISPName, AsCountry: lookupResult.Country, @@ -1406,7 +1414,7 @@ retry: if r.options.JSONOutput && len(scanopts.Hashes) == 0 { scanopts.Hashes = "md5,mmh3,sha256,simhash" } - var hashesMap = map[string]string{} + hashesMap := make(map[string]interface{}) if scanopts.Hashes != "" { hs := strings.Split(scanopts.Hashes, ",") builder.WriteString(" [") @@ -1706,8 +1714,8 @@ func getDNSData(hp *httpx.HTTPX, hostname string) (ips, cnames []string, err err return } -func normalizeHeaders(headers map[string][]string) map[string]string { - normalized := make(map[string]string, len(headers)) +func normalizeHeaders(headers map[string][]string) map[string]interface{} { + normalized := make(map[string]interface{}, len(headers)) for k, v := range headers { normalized[strings.ReplaceAll(strings.ToLower(k), "-", "_")] = strings.Join(v, ", ") } diff --git a/runner/types.go b/runner/types.go index 4fdbb07..ffb4e51 100644 --- a/runner/types.go +++ b/runner/types.go @@ -2,20 +2,21 @@ package runner import ( "fmt" - "reflect" - "strings" "time" + "github.com/bxcodec/faker/v4" + "github.com/bxcodec/faker/v4/pkg/options" "github.com/mitchellh/mapstructure" "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/mapsutil" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" ) type AsnResponse struct { - AsNumber string `json:"as-number" csv:"as-number"` - AsName string `json:"as-name" csv:"as-name"` - AsCountry string `json:"as-country" csv:"as-country"` - AsRange string `json:"as-range" csv:"as-range"` + AsNumber string `json:"as_number" csv:"as_number"` + AsName string `json:"as_name" csv:"as_name"` + AsCountry string `json:"as_country" csv:"as_country"` + AsRange string `json:"as_range" csv:"as_range"` } func (o AsnResponse) String() string { @@ -24,89 +25,86 @@ func (o AsnResponse) String() string { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" dsl:"timestamp"` - ASN interface{} `json:"asn,omitempty" csv:"asn"` + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` + ASN *AsnResponse `json:"asn,omitempty" csv:"asn"` err error - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp" dsl:"csp"` - TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` - Hashes map[string]string `json:"hash,omitempty" csv:"hash" dsl:"hash,header_md5,header_mmh3,header_sha256,header_simhash,body_md5,body_mmh3,body_sha256,body_simhash"` - ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` - CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` - Port string `json:"port,omitempty" csv:"port" dsl:"port"` + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` + TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` + Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` + Port string `json:"port,omitempty" csv:"port"` raw string - URL string `json:"url,omitempty" csv:"url" dsl:"url"` - Input string `json:"input,omitempty" csv:"input" dsl:"input"` - Location string `json:"location,omitempty" csv:"location" dsl:"location"` + URL string `json:"url,omitempty" csv:"url"` + Input string `json:"input,omitempty" csv:"input"` + Location string `json:"location,omitempty" csv:"location"` Title string `json:"title,omitempty" csv:"title"` str string - Scheme string `json:"scheme,omitempty" csv:"scheme" dsl:"scheme"` - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver" dsl:"webserver"` - ResponseBody string `json:"body,omitempty" csv:"body" dsl:"body"` - ContentType string `json:"content_type,omitempty" csv:"content_type" dsl:"content_type"` - Method string `json:"method,omitempty" csv:"method" dsl:"method"` - Host string `json:"host,omitempty" csv:"host" dsl:"host"` - Path string `json:"path,omitempty" csv:"path" dsl:"path"` - FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` - FinalURL string `json:"final_url,omitempty" csv:"final_url"` - ResponseHeader map[string]string `json:"header,omitempty" csv:"header" dsl:"header"` - RawHeader string `json:"raw_header,omitempty" csv:"raw_header"` - Request string `json:"request,omitempty" csv:"request" dsl:"request"` - ResponseTime string `json:"time,omitempty" csv:"time" dsl:"time"` - Jarm string `json:"jarm,omitempty" csv:"jarm"` - ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` - A []string `json:"a,omitempty" csv:"a" dsl:"a"` - CNAMEs []string `json:"cname,omitempty" csv:"cname"` - Technologies []string `json:"tech,omitempty" csv:"tech"` - Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - Words int `json:"words" csv:"words" dsl:"words"` - Lines int `json:"lines" csv:"lines" dsl:"lines"` - StatusCode int `json:"status_code,omitempty" csv:"status_code" dsl:"status_code"` - ContentLength int `json:"content_length,omitempty" csv:"content_length" dsl:"content_length"` - Failed bool `json:"failed" csv:"failed" dsl:"failed"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` + Scheme string `json:"scheme,omitempty" csv:"scheme"` + Error string `json:"error,omitempty" csv:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver"` + ResponseBody string `json:"body,omitempty" csv:"body"` + ContentType string `json:"content_type,omitempty" csv:"content_type"` + Method string `json:"method,omitempty" csv:"method"` + Host string `json:"host,omitempty" csv:"host"` + Path string `json:"path,omitempty" csv:"path"` + FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` + FinalURL string `json:"final_url,omitempty" csv:"final_url"` + ResponseHeader map[string]interface{} `json:"header,omitempty" csv:"header"` + RawHeader string `json:"raw_header,omitempty" csv:"raw_header"` + Request string `json:"request,omitempty" csv:"request"` + ResponseTime string `json:"time,omitempty" csv:"time"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` + ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` + A []string `json:"a,omitempty" csv:"a"` + CNAMEs []string `json:"cname,omitempty" csv:"cname"` + Technologies []string `json:"tech,omitempty" csv:"tech"` + Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` + Words int `json:"words" csv:"words"` + Lines int `json:"lines" csv:"lines"` + StatusCode int `json:"status_code,omitempty" csv:"status_code"` + ContentLength int `json:"content_length,omitempty" csv:"content_length"` + Failed bool `json:"failed" csv:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` } // function to get dsl variables from result struct -func dslVariables() []string { - dslVarList := []string{} - t := reflect.TypeOf(Result{}) - for i := 0; i < t.NumField(); i++ { - field := t.Field(i) - if field.Tag.Get("dsl") != "" { - dslVarList = append(dslVarList, strings.Split(field.Tag.Get("dsl"), ",")...) - } - - if field.Name == "CSPData" { - u := reflect.TypeOf(httpx.CSPData{}) - for j := 0; j < u.NumField(); j++ { - field := u.Field(j) - if field.Tag.Get("dsl") != "" { - dslVarList = append(dslVarList, strings.Split(field.Tag.Get("dsl"), ",")...) - } - } - } +func dslVariables() ([]string, error) { + fakeResult := Result{} + fieldsToIgnore := []string{"Hashes", "ResponseHeader"} + if err := faker.FakeData(&fakeResult, options.WithFieldsToIgnore(fieldsToIgnore...)); err != nil { + return nil, err + } + m, err := ResultToMap(fakeResult) + if err != nil { + return nil, err } - return dslVarList + vars := []string{"header_md5", "header_mmh3", "header_sha256", "header_simhash", "body_md5", "body_mmh3", "body_sha256", "body_simhash"} + mapsutil.Walk(m, func(k string, v any) { + vars = append(vars, k) + }) + + return vars, nil } -func decodeResponse(resp Result, respObj interface{}) error { +func ResultToMap(resp Result) (map[string]any, error) { + m := make(map[string]any) config := &mapstructure.DecoderConfig{ - TagName: "dsl", - Result: &respObj, + TagName: "json", + Result: &m, } decoder, err := mapstructure.NewDecoder(config) if err != nil { - return fmt.Errorf("error creating decoder: %v", err) + return nil, fmt.Errorf("error creating decoder: %v", err) } err = decoder.Decode(resp) if err != nil { - return fmt.Errorf("error decoding: %v", err) + return nil, fmt.Errorf("error decoding: %v", err) } - return nil + return m, nil } From bab68a487f63a921399e8f932493f925f4010c8c Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Thu, 20 Oct 2022 17:54:52 +0530 Subject: [PATCH 278/355] AS input support +asnmap integration (#821) * Add asnmap integration - Use PD's asnmap lib to get asn details * Add AS input support [WIP] * Add AS input support - Add AS Number input support - Update README.md to add AS input example - Add test cases to test `targets()` function - Refactor target function remove if-else with switch case * Resolve merge conflicts * Replace - with _ in asnResponse for json * Replace - with _ in asnResponse for csv --- README.md | 27 +++-- go.mod | 5 +- go.sum | 4 +- runner/runner.go | 75 +++++++----- runner/runner_test.go | 148 ++++++++++++++++++++++ runner/tests/AS14421.txt | 256 +++++++++++++++++++++++++++++++++++++++ runner/types.go | 10 +- 7 files changed, 480 insertions(+), 45 deletions(-) create mode 100644 runner/runner_test.go create mode 100644 runner/tests/AS14421.txt diff --git a/README.md b/README.md index dc7ba2c..5e7548a 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,16 @@ https://173.0.84.6 https://173.0.84.16 https://173.0.84.34 ``` +### AS Number Input +```console +echo AS14421 | httpx -silent +https://216.101.17.248 +https://216.101.17.249 +https://216.101.17.250 +https://216.101.17.251 +https://216.101.17.252 +``` ### Tool Chain @@ -387,15 +396,15 @@ subfinder -d hackerone.com -silent | httpx -asn Use with caution. You are responsible for your actions. Developers assume no liability and are not responsible for any misuse or damage. -https://mta-sts.managed.hackerone.com [AS54113, FASTLY, US, 185.199.108.0/24] -https://gslink.hackerone.com [AS16509, AMAZON-02, US, 13.33.168.0/22] -https://www.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20] -https://mta-sts.forwarding.hackerone.com [AS54113, FASTLY, US, 185.199.108.0/24] -https://resources.hackerone.com [AS16509, AMAZON-02, US, 3.98.0.0/15] -https://support.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.48.0/20] -https://mta-sts.hackerone.com [AS54113, FASTLY, US, 185.199.111.0/24] -https://docs.hackerone.com [AS54113, FASTLY, US, 185.199.109.0/24] -https://api.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20] +https://mta-sts.managed.hackerone.com [AS54113, FASTLY, US] +https://gslink.hackerone.com [AS16509, AMAZON-02, US] +https://www.hackerone.com [AS13335, CLOUDFLARENET, US] +https://mta-sts.forwarding.hackerone.com [AS54113, FASTLY, US] +https://resources.hackerone.com [AS16509, AMAZON-02, US] +https://support.hackerone.com [AS13335, CLOUDFLARENET, US] +https://mta-sts.hackerone.com [AS54113, FASTLY, US] +https://docs.hackerone.com [AS54113, FASTLY, US] +https://api.hackerone.com [AS13335, CLOUDFLARENET, US] ``` diff --git a/go.mod b/go.mod index 0a97d08..3cbd70b 100644 --- a/go.mod +++ b/go.mod @@ -46,15 +46,16 @@ require github.com/spaolacci/murmur3 v1.1.0 require ( github.com/PuerkitoBio/goquery v1.8.0 - github.com/ammario/ipisp/v2 v2.0.0 github.com/bxcodec/faker/v4 v4.0.0-beta.3 github.com/hdm/jarm-go v0.0.7 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/mitchellh/mapstructure v1.5.0 + github.com/projectdiscovery/asnmap v0.0.1 github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655 github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa github.com/projectdiscovery/tlsx v0.0.9 + github.com/stretchr/testify v1.8.0 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 ) @@ -75,6 +76,7 @@ require ( github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae // indirect github.com/cockroachdb/redact v1.0.8 // indirect github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/badger v1.6.2 // indirect github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -88,6 +90,7 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect diff --git a/go.sum b/go.sum index 51c8345..1b8a7d9 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,6 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/ammario/ipisp/v2 v2.0.0 h1:/aRMp5srZViiBfOUGzl/Esqae4s0MDDzm9buhGcZ0XU= -github.com/ammario/ipisp/v2 v2.0.0/go.mod h1:bQ6KAL5LnYYEj6olUn+Bzv/im/4Esa5oGkbv9b+uOjo= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -265,6 +263,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/projectdiscovery/asnmap v0.0.1 h1:n4YCz1ljUaDA3dOUCkjI/bUOtiS7ge1KJ39qpURCd/o= +github.com/projectdiscovery/asnmap v0.0.1/go.mod h1:CjCVDhQPVtmlE247L6YFeIVX9c4m8pOX8V8BmB0JkX8= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK68yPOnNNRmwuXqytl+T9EbwneEUCvMDRjLe0J04= github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= diff --git a/runner/runner.go b/runner/runner.go index 5520788..621eb31 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -24,13 +24,14 @@ import ( "golang.org/x/exp/maps" + asnmap "github.com/projectdiscovery/asnmap/libs" dsl "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" + "github.com/projectdiscovery/mapcidr/asn" "github.com/projectdiscovery/mapsutil" - "github.com/ammario/ipisp/v2" "github.com/bluele/gcache" "github.com/logrusorgru/aurora" "github.com/pkg/errors" @@ -75,12 +76,14 @@ type Runner struct { stats clistats.StatisticsClient ratelimiter ratelimit.Limiter HostErrorsCache gcache.Cache + asnClinet asn.ASNClient } // New creates a new client for running enumeration process. func New(options *Options) (*Runner, error) { runner := &Runner{ - options: options, + options: options, + asnClinet: asn.New(), } var err error if options.TechDetect { @@ -475,11 +478,20 @@ func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int) { return 0 } - expandedTarget := 1 - if iputil.IsCIDR(rawTarget) { + expandedTarget := 0 + switch { + case iputil.IsCIDR(rawTarget): if ipsCount, err := mapcidr.AddressCount(rawTarget); err == nil && ipsCount > 0 { expandedTarget = int(ipsCount) } + case asn.IsASN(rawTarget): + asn := asn.New() + cidrs, _ := asn.GetCIDRsForASNNum(rawTarget) + for _, cidr := range cidrs { + expandedTarget += int(mapcidr.AddressCountIpnet(cidr)) + } + default: + expandedTarget = 1 } return expandedTarget } @@ -910,28 +922,34 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results := make(chan httpx.Target) go func() { defer close(results) - - // A valid target does not contain: - // * - // spaces - if strings.ContainsAny(target, "*") || strings.HasPrefix(target, ".") { + switch { + case strings.ContainsAny(target, "*") || strings.HasPrefix(target, "."): + // A valid target does not contain: + // * + // spaces // trim * and/or . (prefix) from the target to return the domain instead of wilcard target = strings.TrimPrefix(strings.Trim(target, "*"), ".") if !r.testAndSet(target) { return } - } - - // test if the target is a cidr - if iputil.IsCIDR(target) { - cidrIps, err := mapcidr.IPAddresses(target) + results <- httpx.Target{Host: target} + case asn.IsASN(target): + cidrIps, err := r.asnClinet.GetIPAddressesAsStream(target) if err != nil { return } - for _, ip := range cidrIps { + for ip := range cidrIps { results <- httpx.Target{Host: ip} } - } else if r.options.ProbeAllIPS { + case iputil.IsCIDR(target): + cidrIps, err := mapcidr.IPAddressesAsStream(target) + if err != nil { + return + } + for ip := range cidrIps { + results <- httpx.Target{Host: ip} + } + case r.options.ProbeAllIPS: URL, err := urlutil.Parse(target) if err != nil { results <- httpx.Target{Host: target} @@ -943,9 +961,10 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { for _, ip := range ips { results <- httpx.Target{Host: target, CustomIP: ip} } - } else if idxComma := strings.Index(target, ","); idxComma > 0 { + case strings.Index(target, ",") > 0: + idxComma := strings.Index(target, ",") results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]} - } else { + default: results <- httpx.Target{Host: target} } }() @@ -1300,17 +1319,17 @@ retry: var asnResponse *AsnResponse if r.options.Asn { - lookupResult, err := ipisp.LookupIP(context.Background(), net.ParseIP(ip)) - if err != nil { - gologger.Warning().Msg(err.Error()) - } - if lookupResult != nil { - lookupResult.ISPName = stringsutil.TrimSuffixAny(strings.ReplaceAll(lookupResult.ISPName, lookupResult.Country, ""), ", ", " ") + results := asnmap.NewClient().GetData(asnmap.IP(ip)) + if len(results) > 0 { + var cidrs []string + for _, cidr := range asnmap.GetCIDR(results) { + cidrs = append(cidrs, cidr.String()) + } asnResponse = &AsnResponse{ - AsNumber: lookupResult.ASN.String(), - AsName: lookupResult.ISPName, - AsCountry: lookupResult.Country, - AsRange: lookupResult.Range.String(), + AsNumber: fmt.Sprintf("AS%v", results[0].ASN), + AsName: results[0].Org, + AsCountry: results[0].Country, + AsRange: cidrs, } builder.WriteString(" [") if !scanopts.OutputWithNoColor { diff --git a/runner/runner_test.go b/runner/runner_test.go new file mode 100644 index 0000000..534351d --- /dev/null +++ b/runner/runner_test.go @@ -0,0 +1,148 @@ +package runner + +import ( + "os" + "strings" + "testing" + + _ "github.com/projectdiscovery/fdmax/autofdmax" + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/stretchr/testify/require" +) + +func TestRunner_domain_targets(t *testing.T) { + options := &Options{} + r, err := New(options) + require.Nil(t, err, "could not create httpx runner") + input := []string{"example.com", "*.example.com", "example.com,one.one.one.one"} + expected := []httpx.Target{{ + Host: "example.com", + }, { + Host: "example.com", + }, { + Host: "one.one.one.one", + CustomHost: "example.com", + }} + got := []httpx.Target{} + for _, inp := range input { + for target := range r.targets(r.hp, inp) { + got = append(got, target) + } + } + require.ElementsMatch(t, expected, got, "could not exepcted output") +} + +func TestRunner_probeall_targets(t *testing.T) { + options := &Options{ + ProbeAllIPS: true, + } + r, err := New(options) + require.Nil(t, err, "could not create httpx runner") + input := "one.one.one.one" + expected := []httpx.Target{{ + Host: "one.one.one.one", + CustomIP: "2606:4700:4700::1111", + }, + { + Host: "one.one.one.one", + CustomIP: "2606:4700:4700::1001", + }, + { + Host: "one.one.one.one", + CustomIP: "1.0.0.1", + }, + { + Host: "one.one.one.one", + CustomIP: "1.1.1.1", + }} + got := []httpx.Target{} + for target := range r.targets(r.hp, input) { + got = append(got, target) + } + + require.ElementsMatch(t, expected, got, "could not exepcted output") +} + +func TestRunner_cidr_targets(t *testing.T) { + options := &Options{} + r, err := New(options) + require.Nil(t, err, "could not create httpx runner") + input := "173.0.84.0/30" + expected := []httpx.Target{ + { + Host: "173.0.84.0", + }, { + Host: "173.0.84.1", + }, + { + Host: "173.0.84.2", + }, + { + Host: "173.0.84.3", + }} + got := []httpx.Target{} + for target := range r.targets(r.hp, input) { + got = append(got, target) + } + + require.ElementsMatch(t, expected, got, "could not exepcted output") +} + +func TestRunner_asn_targets(t *testing.T) { + options := &Options{} + r, err := New(options) + require.Nil(t, err, "could not create httpx runner") + input := "AS14421" + expected := []httpx.Target{} + expectedOutputFile := "tests/AS14421.txt" + // read the expected IPs from the file + fileContent, err := os.ReadFile(expectedOutputFile) + require.Nil(t, err, "could not read the expectedOutputFile file") + ips := strings.Split(strings.ReplaceAll(string(fileContent), "\r\n", "\n"), "\n") + for _, ip := range ips { + expected = append(expected, httpx.Target{Host: ip}) + } + got := []httpx.Target{} + for target := range r.targets(r.hp, input) { + got = append(got, target) + } + require.ElementsMatch(t, expected, got, "could not exepcted output") +} + +func TestRunner_countTargetFromRawTarget(t *testing.T) { + options := &Options{} + r, err := New(options) + require.Nil(t, err, "could not create httpx runner") + + input := "example.com" + expected := 1 + got := r.countTargetFromRawTarget(input) + require.Equal(t, expected, got, "got wrong output") + + input = "example.com" + expected = 0 + err = r.hm.Set(input, nil) + require.Nil(t, err, "could not set value to hm") + got = r.countTargetFromRawTarget(input) + require.Equal(t, expected, got, "got wrong output") + + input = "" + expected = 0 + got = r.countTargetFromRawTarget(input) + require.Equal(t, expected, got, "got wrong output") + + input = "AS14421" + expected = 256 + got = r.countTargetFromRawTarget(input) + require.Equal(t, expected, got, "got wrong output") + + input = "AS15133" + expected = 153088 + got = r.countTargetFromRawTarget(input) + require.Equal(t, expected, got, "got wrong output") + + input = "173.0.84.0/24" + expected = 256 + got = r.countTargetFromRawTarget(input) + require.Equal(t, expected, got, "got wrong output") +} diff --git a/runner/tests/AS14421.txt b/runner/tests/AS14421.txt new file mode 100644 index 0000000..3d78092 --- /dev/null +++ b/runner/tests/AS14421.txt @@ -0,0 +1,256 @@ +216.101.17.0 +216.101.17.1 +216.101.17.2 +216.101.17.3 +216.101.17.4 +216.101.17.5 +216.101.17.6 +216.101.17.7 +216.101.17.8 +216.101.17.9 +216.101.17.10 +216.101.17.11 +216.101.17.12 +216.101.17.13 +216.101.17.14 +216.101.17.15 +216.101.17.16 +216.101.17.17 +216.101.17.18 +216.101.17.19 +216.101.17.20 +216.101.17.21 +216.101.17.22 +216.101.17.23 +216.101.17.24 +216.101.17.25 +216.101.17.26 +216.101.17.27 +216.101.17.28 +216.101.17.29 +216.101.17.30 +216.101.17.31 +216.101.17.32 +216.101.17.33 +216.101.17.34 +216.101.17.35 +216.101.17.36 +216.101.17.37 +216.101.17.38 +216.101.17.39 +216.101.17.40 +216.101.17.41 +216.101.17.42 +216.101.17.43 +216.101.17.44 +216.101.17.45 +216.101.17.46 +216.101.17.47 +216.101.17.48 +216.101.17.49 +216.101.17.50 +216.101.17.51 +216.101.17.52 +216.101.17.53 +216.101.17.54 +216.101.17.55 +216.101.17.56 +216.101.17.57 +216.101.17.58 +216.101.17.59 +216.101.17.60 +216.101.17.61 +216.101.17.62 +216.101.17.63 +216.101.17.64 +216.101.17.65 +216.101.17.66 +216.101.17.67 +216.101.17.68 +216.101.17.69 +216.101.17.70 +216.101.17.71 +216.101.17.72 +216.101.17.73 +216.101.17.74 +216.101.17.75 +216.101.17.76 +216.101.17.77 +216.101.17.78 +216.101.17.79 +216.101.17.80 +216.101.17.81 +216.101.17.82 +216.101.17.83 +216.101.17.84 +216.101.17.85 +216.101.17.86 +216.101.17.87 +216.101.17.88 +216.101.17.89 +216.101.17.90 +216.101.17.91 +216.101.17.92 +216.101.17.93 +216.101.17.94 +216.101.17.95 +216.101.17.96 +216.101.17.97 +216.101.17.98 +216.101.17.99 +216.101.17.100 +216.101.17.101 +216.101.17.102 +216.101.17.103 +216.101.17.104 +216.101.17.105 +216.101.17.106 +216.101.17.107 +216.101.17.108 +216.101.17.109 +216.101.17.110 +216.101.17.111 +216.101.17.112 +216.101.17.113 +216.101.17.114 +216.101.17.115 +216.101.17.116 +216.101.17.117 +216.101.17.118 +216.101.17.119 +216.101.17.120 +216.101.17.121 +216.101.17.122 +216.101.17.123 +216.101.17.124 +216.101.17.125 +216.101.17.126 +216.101.17.127 +216.101.17.128 +216.101.17.129 +216.101.17.130 +216.101.17.131 +216.101.17.132 +216.101.17.133 +216.101.17.134 +216.101.17.135 +216.101.17.136 +216.101.17.137 +216.101.17.138 +216.101.17.139 +216.101.17.140 +216.101.17.141 +216.101.17.142 +216.101.17.143 +216.101.17.144 +216.101.17.145 +216.101.17.146 +216.101.17.147 +216.101.17.148 +216.101.17.149 +216.101.17.150 +216.101.17.151 +216.101.17.152 +216.101.17.153 +216.101.17.154 +216.101.17.155 +216.101.17.156 +216.101.17.157 +216.101.17.158 +216.101.17.159 +216.101.17.160 +216.101.17.161 +216.101.17.162 +216.101.17.163 +216.101.17.164 +216.101.17.165 +216.101.17.166 +216.101.17.167 +216.101.17.168 +216.101.17.169 +216.101.17.170 +216.101.17.171 +216.101.17.172 +216.101.17.173 +216.101.17.174 +216.101.17.175 +216.101.17.176 +216.101.17.177 +216.101.17.178 +216.101.17.179 +216.101.17.180 +216.101.17.181 +216.101.17.182 +216.101.17.183 +216.101.17.184 +216.101.17.185 +216.101.17.186 +216.101.17.187 +216.101.17.188 +216.101.17.189 +216.101.17.190 +216.101.17.191 +216.101.17.192 +216.101.17.193 +216.101.17.194 +216.101.17.195 +216.101.17.196 +216.101.17.197 +216.101.17.198 +216.101.17.199 +216.101.17.200 +216.101.17.201 +216.101.17.202 +216.101.17.203 +216.101.17.204 +216.101.17.205 +216.101.17.206 +216.101.17.207 +216.101.17.208 +216.101.17.209 +216.101.17.210 +216.101.17.211 +216.101.17.212 +216.101.17.213 +216.101.17.214 +216.101.17.215 +216.101.17.216 +216.101.17.217 +216.101.17.218 +216.101.17.219 +216.101.17.220 +216.101.17.221 +216.101.17.222 +216.101.17.223 +216.101.17.224 +216.101.17.225 +216.101.17.226 +216.101.17.227 +216.101.17.228 +216.101.17.229 +216.101.17.230 +216.101.17.231 +216.101.17.232 +216.101.17.233 +216.101.17.234 +216.101.17.235 +216.101.17.236 +216.101.17.237 +216.101.17.238 +216.101.17.239 +216.101.17.240 +216.101.17.241 +216.101.17.242 +216.101.17.243 +216.101.17.244 +216.101.17.245 +216.101.17.246 +216.101.17.247 +216.101.17.248 +216.101.17.249 +216.101.17.250 +216.101.17.251 +216.101.17.252 +216.101.17.253 +216.101.17.254 +216.101.17.255 \ No newline at end of file diff --git a/runner/types.go b/runner/types.go index ffb4e51..4d7fbe5 100644 --- a/runner/types.go +++ b/runner/types.go @@ -13,14 +13,14 @@ import ( ) type AsnResponse struct { - AsNumber string `json:"as_number" csv:"as_number"` - AsName string `json:"as_name" csv:"as_name"` - AsCountry string `json:"as_country" csv:"as_country"` - AsRange string `json:"as_range" csv:"as_range"` + AsNumber string `json:"as_number" csv:"as_number"` + AsName string `json:"as_name" csv:"as_name"` + AsCountry string `json:"as_country" csv:"as_country"` + AsRange []string `json:"as_range" csv:"as_range"` } func (o AsnResponse) String() string { - return fmt.Sprintf("%v, %v, %v, %v", o.AsNumber, o.AsName, o.AsCountry, o.AsRange) + return fmt.Sprintf("%v, %v, %v", o.AsNumber, o.AsName, o.AsCountry) } // Result of a scan From 7d7807c8327d1156c737b2e986e7e559fcbb3510 Mon Sep 17 00:00:00 2001 From: lucy9x Date: Mon, 24 Oct 2022 13:33:20 +0700 Subject: [PATCH 279/355] add OnResultCallback options --- go.mod | 12 ++++++------ go.sum | 22 ++++++++++++++-------- runner/options.go | 1 + runner/runner.go | 6 +++++- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 5a389cd..1f5c110 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/microcosm-cc/bluemonday v1.0.19 + github.com/microcosm-cc/bluemonday v1.0.20 github.com/miekg/dns v1.1.48 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 @@ -18,9 +18,9 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c + github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.0.9 + github.com/projectdiscovery/goflags v0.1.1 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 - github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 + github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 github.com/projectdiscovery/wappalyzergo v0.0.52 github.com/remeh/sizedwaitgroup v1.0.0 @@ -38,8 +38,8 @@ require ( go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect + golang.org/x/net v0.0.0-20220909164309-bea034e7d591 + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect golang.org/x/text v0.3.7 ) diff --git a/go.sum b/go.sum index 78006ce..1e57c68 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= -github.com/microcosm-cc/bluemonday v1.0.19 h1:OI7hoF5FY4pFz2VA//RN8TfM0YJ2dJcl4P4APrCWy6c= -github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= +github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= +github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= @@ -135,14 +135,15 @@ github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIre github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483/go.mod h1:wjS/oBWbzlayJ/aTK0KW0oOHGO03G8oEYzuN6stI8Ho= -github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c h1:/+xJK8e/Gj/zAmudWh5l2SzGJB+CkwYnraelCkBe7Aw= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= +github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 h1:4o97N9ftX1J3iKlIRVMPVOVZs4qbCczJvoFF2WA40t4= +github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= -github.com/projectdiscovery/goflags v0.0.9 h1:bPsYIPE1LvdgYaM3XNX0YmS68e6huv22W22rKh5IscI= -github.com/projectdiscovery/goflags v0.0.9/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI= +github.com/projectdiscovery/goflags v0.1.1 h1:AEtT14D9OC10HWyZwDQaSLjuK8ZKoBrSYlsLItvMKZI= +github.com/projectdiscovery/goflags v0.1.1/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -183,8 +184,9 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3 h1:EbP+rVR7NPsFKfwhzshmXP2GMjqBpaMZurJzQ5PYvbY= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= +github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ= +github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= github.com/projectdiscovery/wappalyzergo v0.0.52 h1:AbM+4KEikwgxpeoWg4Gf4KPxkXAeLHSpWyUaNcaW+sQ= @@ -218,6 +220,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -279,8 +282,10 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -313,8 +318,9 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/runner/options.go b/runner/options.go index 53d7990..5afadca 100644 --- a/runner/options.go +++ b/runner/options.go @@ -244,6 +244,7 @@ type Options struct { OutputMatchResponseTime string OutputFilterResponseTime string HealthCheck bool + OnResult func(result Result) } // ParseOptions parses the command line options for application diff --git a/runner/runner.go b/runner/runner.go index 19471b1..ba0199b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1491,7 +1491,7 @@ retry: chainItems = append(chainItems, resp.GetChainAsSlice()...) } - return Result{ + result := Result{ Timestamp: time.Now(), Request: request, ResponseHeader: responseHeader, @@ -1535,6 +1535,10 @@ retry: ASN: asnResponse, ExtractRegex: extractRegex, } + if r.options.OnResult != nil { + r.options.OnResult(result) + } + return result } // SaveResumeConfig to file From 3fe1cc2268d7e345a2217d67af63741d38e05eb4 Mon Sep 17 00:00:00 2001 From: lucy9x Date: Mon, 24 Oct 2022 13:41:32 +0700 Subject: [PATCH 280/355] add OnResultCallback options --- go.mod | 19 +++++++++--------- go.sum | 61 ++++++++++++++++++++++++++++------------------------------ 2 files changed, 38 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 1f5c110..be0c05d 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,12 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.20 - github.com/miekg/dns v1.1.48 // indirect + github.com/miekg/dns v1.1.50 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 + github.com/projectdiscovery/fastdialer v0.0.17 github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c @@ -24,11 +24,11 @@ require ( github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 - github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 + github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 github.com/projectdiscovery/mapcidr v1.0.1 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d - github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 + github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 github.com/projectdiscovery/wappalyzergo v0.0.52 @@ -39,7 +39,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 golang.org/x/net v0.0.0-20220909164309-bea034e7d591 - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/sys v0.0.0-20220731174439-a90be440212d // indirect golang.org/x/text v0.3.7 ) @@ -65,10 +65,10 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d // indirect + github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect github.com/projectdiscovery/networkpolicy v0.0.1 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect - github.com/projectdiscovery/retryabledns v1.0.13 // indirect + github.com/projectdiscovery/retryabledns v1.0.16 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect @@ -77,9 +77,8 @@ require ( github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect - golang.org/x/mod v0.4.2 // indirect - golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/tools v0.1.12 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1e57c68..bcb4038 100644 --- a/go.sum +++ b/go.sum @@ -84,14 +84,12 @@ github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/z github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= -github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= -github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -120,27 +118,24 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= -github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d h1:1Q94movG4VMVo+qf781ttXXt2bxBOIGMenmuCAJDLfE= -github.com/projectdiscovery/blackrock v0.0.0-20210903102120-5a9d2412d21d/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= +github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK68yPOnNNRmwuXqytl+T9EbwneEUCvMDRjLe0J04= +github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 h1:uxOMKHOajVaiz/pMUzc3nvWhICa/euZN+SoVzpqcPiE= -github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= +github.com/projectdiscovery/fastdialer v0.0.17 h1:wT7LinZAcyEzkfhJLhWNQrA9m79+hTd6CJ7aWTJfXjk= +github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= -github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/fileutil v0.0.0-20220506114156-c4ab20801483/go.mod h1:wjS/oBWbzlayJ/aTK0KW0oOHGO03G8oEYzuN6stI8Ho= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963 h1:4o97N9ftX1J3iKlIRVMPVOVZs4qbCczJvoFF2WA40t4= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.1.1 h1:AEtT14D9OC10HWyZwDQaSLjuK8ZKoBrSYlsLItvMKZI= github.com/projectdiscovery/goflags v0.1.1/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= @@ -154,14 +149,10 @@ github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:Gzruq github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= -github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1:blmYJkS8lSrrx3QcmcgS2tZIxlojeVmoGeA9twslCBU= -github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619 h1:PSYGmqVDxBe9Pnbe14MIX7NXTHJqxMwBLzwDNBwSZYk= -github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619/go.mod h1:w5bXH/PHqM8/vF0bFN6fZMggm7IlPKK2dJdz5McY+Qw= +github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 h1:HRqev12wKvcwK1fe4pSlMfQdPHo9LfTxuFeRN4f3tS4= +github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1:vHRC+9exsfSbEngMKDl0xiWqkxlLk3lHQZpbS2yFT8U= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= -github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v1.0.0/go.mod h1:5QkKrV6rNQQurCZI3nNedFsAOYp04mRDkC5yht+znYA= github.com/projectdiscovery/mapcidr v1.0.1 h1:eaLBRrImwlYXv8vbXTwR4sxoQqIxR3Y5k/Sd7HhTIII= github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= @@ -170,21 +161,21 @@ github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpG github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= -github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/go.mod h1:tXaLDs4n3pRZHwfa8mdXpUWe/AYDNK3HlWDjldhRbjI= -github.com/projectdiscovery/retryabledns v1.0.13 h1:Ogfv0fl3Nszb+Nq2S2qQmE+PJDlStVxyLkmTotMohZY= -github.com/projectdiscovery/retryabledns v1.0.13/go.mod h1:EeqHcAPp0g2GljT4qkxKSAE47Dj0ZrJQ46R9ct3Muhk= +github.com/projectdiscovery/retryabledns v1.0.15/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk= +github.com/projectdiscovery/retryabledns v1.0.16 h1:yn5+W4roniHnte41mWkZO2mYV4plvasDuJLvotB2nH0= +github.com/projectdiscovery/retryabledns v1.0.16/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= -github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6 h1:zZmLOwj2duDafEO9i1KqmHMW0SL9+Wwmco9sxhBCVwQ= github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA= +github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= +github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= +github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ= github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= @@ -234,6 +225,7 @@ github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+Seva github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= @@ -256,12 +248,14 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -281,8 +275,9 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= @@ -290,8 +285,9 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -310,17 +306,17 @@ golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80= +golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -334,13 +330,14 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 h1:BonxutuHCTL0rBDnZlKjpGIQFTjyUVTexFOdWkB6Fg0= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From a90e5a140c49338b1353e992348f16cc44a3e8de Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 24 Oct 2022 10:00:54 +0200 Subject: [PATCH 281/355] minor signature change --- runner/options.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runner/options.go b/runner/options.go index 5afadca..b15e32e 100644 --- a/runner/options.go +++ b/runner/options.go @@ -34,6 +34,9 @@ const ( var defaultProviders = strings.Join(cdncheck.GetDefaultProviders(), ", ") +// OnResultCallback (hostResult) +type OnResultCallback func(Result) + type scanOptions struct { Methods []string StoreResponseDirectory string @@ -244,7 +247,7 @@ type Options struct { OutputMatchResponseTime string OutputFilterResponseTime string HealthCheck bool - OnResult func(result Result) + OnResult OnResultCallback } // ParseOptions parses the command line options for application From 61f0580c4fadb00356807a57a17af8934d4aaadc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Oct 2022 10:05:58 +0000 Subject: [PATCH 282/355] chore(deps): bump golangci/golangci-lint-action from 3.2.0 to 3.3.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.2.0...v3.3.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 98ecbd6..d6921ef 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -16,7 +16,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.2.0 + uses: golangci/golangci-lint-action@v3.3.0 with: version: latest args: --timeout 5m From 49ca8778cd63495c845ee7939c5542d901c67749 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Oct 2022 10:11:46 +0000 Subject: [PATCH 283/355] chore(deps): bump github.com/projectdiscovery/fileutil Bumps [github.com/projectdiscovery/fileutil](https://github.com/projectdiscovery/fileutil) from 0.0.1 to 0.0.2. - [Release notes](https://github.com/projectdiscovery/fileutil/releases) - [Commits](https://github.com/projectdiscovery/fileutil/compare/v0.0.1...v0.0.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fileutil dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3cbd70b..fc1551e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/fastdialer v0.0.17 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.1 + github.com/projectdiscovery/fileutil v0.0.2 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.1 github.com/projectdiscovery/gologger v1.1.4 diff --git a/go.sum b/go.sum index 1b8a7d9..c0b1d9e 100644 --- a/go.sum +++ b/go.sum @@ -284,8 +284,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= -github.com/projectdiscovery/fileutil v0.0.1 h1:3K3UqCDOan3LsvWhV0nyvVuMWSwCloNPUJIGcXsi1os= -github.com/projectdiscovery/fileutil v0.0.1/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2LYinWCSEmzZOQsJCLg= +github.com/projectdiscovery/fileutil v0.0.2 h1:UI4NuBeDECIfGz78rHM5Wos4VClSuxEP7L0e27Lor3c= +github.com/projectdiscovery/fileutil v0.0.2/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2LYinWCSEmzZOQsJCLg= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= From 52a92829bc57b559cd80aa734e0babdf80f0d5ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Oct 2022 10:11:51 +0000 Subject: [PATCH 284/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.65 to 0.0.66. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.65...v0.0.66) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3cbd70b..019e1f2 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/stringsutil v0.0.1 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.65 + github.com/projectdiscovery/wappalyzergo v0.0.66 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 1b8a7d9..aded6a5 100644 --- a/go.sum +++ b/go.sum @@ -341,8 +341,8 @@ github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjb github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.65 h1:M1/r75lJI5rezDmfrARQiNNm+Ii9o3jsV+Q+F69Lcso= -github.com/projectdiscovery/wappalyzergo v0.0.65/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.66 h1:+egr7a8VRNtSeqDDap3IYPW2rH+sCpB9Irwp94ipA7E= +github.com/projectdiscovery/wappalyzergo v0.0.66/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= From 67f8a93655e0edcc86622862c98a51a634c802bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Oct 2022 10:11:56 +0000 Subject: [PATCH 285/355] chore(deps): bump github.com/projectdiscovery/stringsutil Bumps [github.com/projectdiscovery/stringsutil](https://github.com/projectdiscovery/stringsutil) from 0.0.1 to 0.0.2. - [Release notes](https://github.com/projectdiscovery/stringsutil/releases) - [Commits](https://github.com/projectdiscovery/stringsutil/compare/v0.0.1...v0.0.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/stringsutil dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3cbd70b..37b2f09 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 - github.com/projectdiscovery/stringsutil v0.0.1 + github.com/projectdiscovery/stringsutil v0.0.2 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 github.com/projectdiscovery/wappalyzergo v0.0.65 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 1b8a7d9..b7f68de 100644 --- a/go.sum +++ b/go.sum @@ -217,7 +217,6 @@ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/p github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= -github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -335,8 +334,8 @@ github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= -github.com/projectdiscovery/stringsutil v0.0.1 h1:a6TCMT+D1aUsoZxNiYf9O30wiDOoLOHDwj89HBjr5BQ= -github.com/projectdiscovery/stringsutil v0.0.1/go.mod h1:TDi2LEqR3OML0BxGoMbbfAHSk5AdfHX762Oc302sgmM= +github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= +github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= @@ -490,7 +489,7 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= From c3c1ca7330461dedf790c790d005c766557bcf83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Oct 2022 10:11:59 +0000 Subject: [PATCH 286/355] chore(deps): bump github.com/stretchr/testify from 1.8.0 to 1.8.1 Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.0 to 1.8.1. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.0...v1.8.1) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3cbd70b..83891f9 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655 github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa github.com/projectdiscovery/tlsx v0.0.9 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 ) diff --git a/go.sum b/go.sum index 1b8a7d9..98ba387 100644 --- a/go.sum +++ b/go.sum @@ -377,6 +377,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -386,8 +387,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= From 25122cee34fc50e35c01ffb4a51438a05be8c1c8 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 25 Oct 2022 03:22:17 +0200 Subject: [PATCH 287/355] fixing func signature --- common/httpx/title.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/httpx/title.go b/common/httpx/title.go index 7ad7cb1..4f799a8 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -35,7 +35,7 @@ func ExtractTitle(r *Response) (title string) { // remove unwanted chars title = strings.TrimSpace(strings.Trim(title, cutset)) - title = stringsutil.ReplaceAny(title, "\n", "\r") + title = stringsutil.ReplaceAll(title, "\n", "\r") return title } From 7e1072ab4a9b05e8d5a3a5edeffbc6367db9c740 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Oct 2022 10:11:00 +0000 Subject: [PATCH 288/355] chore(deps): bump github.com/projectdiscovery/fileutil Bumps [github.com/projectdiscovery/fileutil](https://github.com/projectdiscovery/fileutil) from 0.0.2 to 0.0.3. - [Release notes](https://github.com/projectdiscovery/fileutil/releases) - [Commits](https://github.com/projectdiscovery/fileutil/compare/v0.0.2...v0.0.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fileutil dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fcb5025..bb2821e 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/fastdialer v0.0.17 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.2 + github.com/projectdiscovery/fileutil v0.0.3 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.1 github.com/projectdiscovery/gologger v1.1.4 diff --git a/go.sum b/go.sum index a0c2c71..6273c63 100644 --- a/go.sum +++ b/go.sum @@ -283,8 +283,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= -github.com/projectdiscovery/fileutil v0.0.2 h1:UI4NuBeDECIfGz78rHM5Wos4VClSuxEP7L0e27Lor3c= -github.com/projectdiscovery/fileutil v0.0.2/go.mod h1:Oo6ZEvXmQz/xPF0YukzmwpdW2LYinWCSEmzZOQsJCLg= +github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= +github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= @@ -546,6 +546,7 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From faaada361f8846a4147b99b4fbe36d838bea2ccf Mon Sep 17 00:00:00 2001 From: Sajad Date: Thu, 27 Oct 2022 14:25:53 +0530 Subject: [PATCH 289/355] add stored_response_path to output json (#832) * add stored_response_path to output json * path -> filepath --- runner/runner.go | 93 ++++++++++++++++++++++++------------------------ runner/types.go | 93 ++++++++++++++++++++++++------------------------ 2 files changed, 94 insertions(+), 92 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 0e8906a..89e8a83 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -13,7 +13,6 @@ import ( "net/http/httputil" "net/url" "os" - "path" "path/filepath" "reflect" "regexp" @@ -1508,6 +1507,7 @@ retry: } // store responses or chain in directory + var responsePath string if scanopts.StoreResponse || scanopts.StoreChain { domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".") @@ -1520,7 +1520,7 @@ retry: domainFile = strings.ReplaceAll(domainFile, "/", "[slash]") + ".txt" // store response - responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile) + responsePath = filepath.Join(scanopts.StoreResponseDirectory, domainFile) respRaw := resp.Raw if len(respRaw) > scanopts.MaxResponseBodySizeToSave { respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave] @@ -1531,7 +1531,7 @@ retry: } if scanopts.StoreChain && resp.HasChain() { domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") - responsePath := path.Join(scanopts.StoreResponseDirectory, domainFile) + responsePath = filepath.Join(scanopts.StoreResponseDirectory, domainFile) writeErr := os.WriteFile(responsePath, []byte(resp.GetChain()), 0644) if writeErr != nil { gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) @@ -1566,49 +1566,50 @@ retry: } result := Result{ - Timestamp: time.Now(), - Request: request, - ResponseHeader: responseHeader, - RawHeader: rawResponseHeader, - Scheme: parsed.Scheme, - Port: finalPort, - Path: finalPath, - raw: resp.Raw, - URL: fullURL, - Input: origInput, - ContentLength: resp.ContentLength, - ChainStatusCodes: chainStatusCodes, - Chain: chainItems, - StatusCode: resp.StatusCode, - Location: resp.GetHeaderPart("Location", ";"), - ContentType: resp.GetHeaderPart("Content-Type", ";"), - Title: title, - str: builder.String(), - VHost: isvhost, - WebServer: serverHeader, - ResponseBody: serverResponseRaw, - WebSocket: isWebSocket, - TLSData: resp.TLSData, - CSPData: resp.CSPData, - Pipeline: pipeline, - HTTP2: http2, - Method: method, - Host: ip, - A: ips, - CNAMEs: cnames, - CDN: isCDN, - CDNName: cdnName, - ResponseTime: resp.Duration.String(), - Technologies: technologies, - FinalURL: finalURL, - FavIconMMH3: faviconMMH3, - Hashes: hashesMap, - Extracts: extractResult, - Jarm: jarmhash, - Lines: resp.Lines, - Words: resp.Words, - ASN: asnResponse, - ExtractRegex: extractRegex, + Timestamp: time.Now(), + Request: request, + ResponseHeader: responseHeader, + RawHeader: rawResponseHeader, + Scheme: parsed.Scheme, + Port: finalPort, + Path: finalPath, + raw: resp.Raw, + URL: fullURL, + Input: origInput, + ContentLength: resp.ContentLength, + ChainStatusCodes: chainStatusCodes, + Chain: chainItems, + StatusCode: resp.StatusCode, + Location: resp.GetHeaderPart("Location", ";"), + ContentType: resp.GetHeaderPart("Content-Type", ";"), + Title: title, + str: builder.String(), + VHost: isvhost, + WebServer: serverHeader, + ResponseBody: serverResponseRaw, + WebSocket: isWebSocket, + TLSData: resp.TLSData, + CSPData: resp.CSPData, + Pipeline: pipeline, + HTTP2: http2, + Method: method, + Host: ip, + A: ips, + CNAMEs: cnames, + CDN: isCDN, + CDNName: cdnName, + ResponseTime: resp.Duration.String(), + Technologies: technologies, + FinalURL: finalURL, + FavIconMMH3: faviconMMH3, + Hashes: hashesMap, + Extracts: extractResult, + Jarm: jarmhash, + Lines: resp.Lines, + Words: resp.Words, + ASN: asnResponse, + ExtractRegex: extractRegex, + StoredResponsePath: responsePath, } if r.options.OnResult != nil { r.options.OnResult(result) diff --git a/runner/types.go b/runner/types.go index 4d7fbe5..77e5b96 100644 --- a/runner/types.go +++ b/runner/types.go @@ -25,52 +25,53 @@ func (o AsnResponse) String() string { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` - ASN *AsnResponse `json:"asn,omitempty" csv:"asn"` - err error - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` - Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"` - ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` - CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` - Port string `json:"port,omitempty" csv:"port"` - raw string - URL string `json:"url,omitempty" csv:"url"` - Input string `json:"input,omitempty" csv:"input"` - Location string `json:"location,omitempty" csv:"location"` - Title string `json:"title,omitempty" csv:"title"` - str string - Scheme string `json:"scheme,omitempty" csv:"scheme"` - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"body,omitempty" csv:"body"` - ContentType string `json:"content_type,omitempty" csv:"content_type"` - Method string `json:"method,omitempty" csv:"method"` - Host string `json:"host,omitempty" csv:"host"` - Path string `json:"path,omitempty" csv:"path"` - FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` - FinalURL string `json:"final_url,omitempty" csv:"final_url"` - ResponseHeader map[string]interface{} `json:"header,omitempty" csv:"header"` - RawHeader string `json:"raw_header,omitempty" csv:"raw_header"` - Request string `json:"request,omitempty" csv:"request"` - ResponseTime string `json:"time,omitempty" csv:"time"` - Jarm string `json:"jarm,omitempty" csv:"jarm"` - ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` - A []string `json:"a,omitempty" csv:"a"` - CNAMEs []string `json:"cname,omitempty" csv:"cname"` - Technologies []string `json:"tech,omitempty" csv:"tech"` - Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - Words int `json:"words" csv:"words"` - Lines int `json:"lines" csv:"lines"` - StatusCode int `json:"status_code,omitempty" csv:"status_code"` - ContentLength int `json:"content_length,omitempty" csv:"content_length"` - Failed bool `json:"failed" csv:"failed"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` + ASN *AsnResponse `json:"asn,omitempty" csv:"asn"` + err error + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` + TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` + Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` + Port string `json:"port,omitempty" csv:"port"` + raw string + URL string `json:"url,omitempty" csv:"url"` + Input string `json:"input,omitempty" csv:"input"` + Location string `json:"location,omitempty" csv:"location"` + Title string `json:"title,omitempty" csv:"title"` + str string + Scheme string `json:"scheme,omitempty" csv:"scheme"` + Error string `json:"error,omitempty" csv:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver"` + ResponseBody string `json:"body,omitempty" csv:"body"` + ContentType string `json:"content_type,omitempty" csv:"content_type"` + Method string `json:"method,omitempty" csv:"method"` + Host string `json:"host,omitempty" csv:"host"` + Path string `json:"path,omitempty" csv:"path"` + FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` + FinalURL string `json:"final_url,omitempty" csv:"final_url"` + ResponseHeader map[string]interface{} `json:"header,omitempty" csv:"header"` + RawHeader string `json:"raw_header,omitempty" csv:"raw_header"` + Request string `json:"request,omitempty" csv:"request"` + ResponseTime string `json:"time,omitempty" csv:"time"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` + ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` + A []string `json:"a,omitempty" csv:"a"` + CNAMEs []string `json:"cname,omitempty" csv:"cname"` + Technologies []string `json:"tech,omitempty" csv:"tech"` + Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` + Words int `json:"words" csv:"words"` + Lines int `json:"lines" csv:"lines"` + StatusCode int `json:"status_code,omitempty" csv:"status_code"` + ContentLength int `json:"content_length,omitempty" csv:"content_length"` + Failed bool `json:"failed" csv:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` + StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path"` } // function to get dsl variables from result struct From 8af2f59d9646a3e530dcef82b29c33b19cffa996 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 27 Oct 2022 11:04:22 +0200 Subject: [PATCH 290/355] Fixing https with burp (#753) * Fixing https with burp * solving by making error non-fatal * adding missing error comparison --- common/httpx/httpx.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index c524ade..81e0990 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -113,6 +113,7 @@ func New(options *Options) (*HTTPX, error) { }, DisableKeepAlives: true, } + if httpx.Options.SniName != "" { transport.TLSClientConfig.ServerName = httpx.Options.SniName } @@ -165,13 +166,13 @@ func (h *HTTPX) Do(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*Re var gzipRetry bool get_response: httpresp, err := h.getResponse(req, unsafeOptions) - if err != nil { + if httpresp == nil && err != nil { return nil, err } var shouldIgnoreErrors, shouldIgnoreBodyErrors bool switch { - case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny("i/o timeout"): + case h.Options.Unsafe && req.Method == http.MethodHead && !stringsutil.ContainsAny(err.Error(), "i/o timeout"): shouldIgnoreErrors = true shouldIgnoreBodyErrors = true } @@ -183,6 +184,11 @@ get_response: // httputil.DumpResponse does not handle websockets headers, rawResp, err := pdhttputil.DumpResponseHeadersAndRaw(httpresp) if err != nil { + if stringsutil.ContainsAny(err.Error(), "tls: user canceled") { + shouldIgnoreErrors = true + shouldIgnoreBodyErrors = true + } + // Edge case - some servers respond with gzip encoding header but uncompressed body, in this case the standard library configures the reader as gzip, triggering an error when read. // The bytes slice is not accessible because of abstraction, therefore we need to perform the request again tampering the Accept-Encoding header if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") { @@ -196,7 +202,6 @@ get_response: } resp.Raw = string(rawResp) resp.RawHeaders = string(headers) - var respbody []byte // websockets don't have a readable body if httpresp.StatusCode != http.StatusSwitchingProtocols { @@ -274,11 +279,10 @@ type UnsafeOptions struct { } // getResponse returns response from safe / unsafe request -func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (*http.Response, error) { +func (h *HTTPX) getResponse(req *retryablehttp.Request, unsafeOptions UnsafeOptions) (resp *http.Response, err error) { if h.Options.Unsafe { return h.doUnsafeWithOptions(req, unsafeOptions) } - return h.client.Do(req) } From e7ab403b33ba4e6a121f7be8ef22e9c232b27574 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 27 Oct 2022 11:52:01 +0200 Subject: [PATCH 291/355] readme + workflow update --- .github/workflows/release-binary.yml | 1 + .goreleaser.yml | 7 ++++++ README.md | 37 +++++++++++++++------------- runner/banner.go | 4 +-- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 855356f..028de6e 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -27,3 +27,4 @@ jobs: workdir: . env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" diff --git a/.goreleaser.yml b/.goreleaser.yml index 88ba796..7d5bd2d 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -33,3 +33,10 @@ archives: checksum: algorithm: sha256 + +announce: + slack: + enabled: true + channel: '#release' + username: GoReleaser + message_template: '{{ .ProjectName }} {{ .Tag }} is out! Check it out at {{ .ReleaseURL }}' \ No newline at end of file diff --git a/README.md b/README.md index 5e7548a..06c96ee 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Flags: INPUT: -l, -list string input file containing list of hosts to process -rr, -request string file containing raw request + -u, -target string[] input target host(s) to probe PROBES: -sc, -status-code display response status-code @@ -109,7 +110,6 @@ PROBES: -probe display probe status MATCHERS: - -mdc -match-condition string match response with dsl expression condition -mc, -match-code string match response with specified status code (-mc 200,302) -ml, -match-length string match response with specified content length (-ml 100,102) -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) @@ -117,15 +117,15 @@ MATCHERS: -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string match response with specified string (-ms admin) -mr, -match-regex string match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (azure, cloudflare, cloudfront, fastly, incapsula, oracle, google, akamai, sucuri, leaseweb) + -mcdn, -match-cdn string[] match host with specified cdn provider (fastly, incapsula, oracle, google, azure, cloudflare, cloudfront, sucuri, leaseweb, akamai) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') + -mdc, -match-condition string match response with dsl expression condition EXTRACTOR: - -er, -extract-regex string[] Display response content with matched regex - -ep, -extract-preset string[] Display response content with matched preset regex + -er, -extract-regex string[] display response content with matched regex + -ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail) FILTERS: - -fdc -filter-condition string filter response with dsl expression condition -fc, -filter-code string filter response with specified status code (-fc 403,401) -fl, -filter-length string filter response with specified content length (-fl 23,33) -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) @@ -133,8 +133,9 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) -fs, -filter-string string filter response with specified string (-fs admin) -fe, -filter-regex string filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (azure, cloudflare, cloudfront, fastly, incapsula, oracle, google, akamai, sucuri, leaseweb) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (fastly, incapsula, oracle, google, azure, cloudflare, cloudfront, sucuri, leaseweb, akamai) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') + -fdc, -filter-condition string filter response with dsl expression condition RATE-LIMIT: -t, -threads int number of threads to use (default 50) @@ -142,15 +143,16 @@ RATE-LIMIT: -rlm, -rate-limit-minute int maximum number of requests to send per minute MISCELLANEOUS: - -pa, -probe-all-ips probe all the ips associated with same host - -p, -ports string[] ports to probe (nmap syntax: eg 1,2-10,11) - -path string path or list of paths to probe (comma-separated, file) - -tls-probe send http probes on the extracted TLS domains (dns_name) - -csp-probe send http probes on the extracted CSP domains - -tls-grab perform TLS(SSL) data grabbing - -pipeline probe and display server supporting HTTP1.1 pipeline - -http2 probe and display server supporting HTTP2 - -vhost probe and display server supporting VHOST + -pa, -probe-all-ips probe all the ips associated with same host + -p, -ports string[] ports to probe (nmap syntax: eg http:1,2-10,11,https:80) + -path string path or list of paths to probe (comma-separated, file) + -tls-probe send http probes on the extracted TLS domains (dns_name) + -csp-probe send http probes on the extracted CSP domains + -tls-grab perform TLS(SSL) data grabbing + -pipeline probe and display server supporting HTTP1.1 pipeline + -http2 probe and display server supporting HTTP2 + -vhost probe and display server supporting VHOST + -ldv, -list-dsl-variables list json output field keys name that support dsl matcher/filter OUTPUT: -o, -output string file to write output results @@ -166,8 +168,8 @@ CONFIGURATIONS: -r, -resolvers string[] list of custom resolver (file or comma separated) -allow string[] allowed list of IP/CIDR's to process (file or comma separated) -deny string[] denied list of IP/CIDR's to process (file or comma separated) - -sni, -sni-name string Custom TLS SNI name - -random-agent Enable Random User-Agent to use (default true) + -sni, -sni-name string custom TLS SNI name + -random-agent enable Random User-Agent to use (default true) -H, -header string[] custom http headers to send with request -http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080) -unsafe send raw requests skipping golang normalization @@ -189,6 +191,7 @@ DEBUG: -debug-resp display response content in cli -version display httpx version -stats display scan statistic + -profile-mem string optional httpx memory profile dump file -silent silent mode -v, -verbose verbose mode -si, -stats-interval int number of seconds to wait between showing a statistics update (default: 5) diff --git a/runner/banner.go b/runner/banner.go index 5f64df5..c273f02 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.4 + /_/ v1.2.5 ` // Version is the current version of httpx -const Version = `v1.2.4` +const Version = `v1.2.5` // showBanner is used to show the banner to the user func showBanner() { From 5f5c4f0f57c3ecb6d37be4ff7ac3f3cfe7b773fe Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 27 Oct 2022 11:56:05 +0200 Subject: [PATCH 292/355] projectdiscovery/ratelimit update --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bb2821e..3e51c64 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/projectdiscovery/asnmap v0.0.1 github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655 - github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa + github.com/projectdiscovery/ratelimit v0.0.1 github.com/projectdiscovery/tlsx v0.0.9 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.8.0 diff --git a/go.sum b/go.sum index 6273c63..2e5a898 100644 --- a/go.sum +++ b/go.sum @@ -315,6 +315,8 @@ github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa h1:XTRcow+zyZUYxOlLsSp1AdM4Jl+i/OpPbOZPsfU/IAI= github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= +github.com/projectdiscovery/ratelimit v0.0.1 h1:GnCfbKmkLdDLXT3QS4KS0zCsuDGkoRQE0YDbTqzQmS8= +github.com/projectdiscovery/ratelimit v0.0.1/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= From 18896fb9d90e9e4b4d8581f8bf2590a1e7b960e3 Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Thu, 27 Oct 2022 17:27:40 +0530 Subject: [PATCH 293/355] resolve issue 837 failing test (#838) Removed the failing asn IP count test case --- runner/runner_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/runner/runner_test.go b/runner/runner_test.go index 534351d..1b9abc7 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -136,11 +136,6 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) { got = r.countTargetFromRawTarget(input) require.Equal(t, expected, got, "got wrong output") - input = "AS15133" - expected = 153088 - got = r.countTargetFromRawTarget(input) - require.Equal(t, expected, got, "got wrong output") - input = "173.0.84.0/24" expected = 256 got = r.countTargetFromRawTarget(input) From 7a9a3fd5935804e2ec624d6d01e9ab493be6a1e6 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 28 Oct 2022 16:24:13 +0200 Subject: [PATCH 294/355] fixing lint errors --- runner/options.go | 6 +++--- runner/runner.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/runner/options.go b/runner/options.go index e6d5e88..86971c7 100644 --- a/runner/options.go +++ b/runner/options.go @@ -444,16 +444,16 @@ func ParseOptions() *Options { func (options *Options) ValidateOptions() error { if options.InputFile != "" && !fileutilz.FileNameIsGlob(options.InputFile) && !fileutil.FileExists(options.InputFile) { - return fmt.Errorf("File '%s' does not exist.", options.InputFile) + return fmt.Errorf("file '%s' does not exist", options.InputFile) } if options.InputRawRequest != "" && !fileutil.FileExists(options.InputRawRequest) { - return fmt.Errorf("File '%s' does not exist.", options.InputRawRequest) + return fmt.Errorf("file '%s' does not exist", options.InputRawRequest) } multiOutput := options.CSVOutput && options.JSONOutput if multiOutput { - return fmt.Errorf("Results can only be displayed in one format: 'JSON' or 'CSV'") + return fmt.Errorf("results can only be displayed in one format: 'JSON' or 'CSV'") } if options.Silent { diff --git a/runner/runner.go b/runner/runner.go index 89e8a83..b2f06e4 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -295,9 +295,9 @@ func New(options *Options) (*Runner, error) { runner.hm = hm if options.RateLimitMinute > 0 { - runner.ratelimiter = *ratelimit.New(context.Background(), options.RateLimitMinute, time.Minute) + runner.ratelimiter = *ratelimit.New(context.Background(), int64(options.RateLimitMinute), time.Minute) } else if options.RateLimit > 0 { - runner.ratelimiter = *ratelimit.New(context.Background(), options.RateLimit, time.Second) + runner.ratelimiter = *ratelimit.New(context.Background(), int64(options.RateLimit), time.Second) } else { runner.ratelimiter = *ratelimit.NewUnlimited(context.Background()) } From 33a8fc08feec2468d5ecd3196c5772e4b4623305 Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Sat, 29 Oct 2022 14:36:39 +0530 Subject: [PATCH 295/355] Append to file if output file is already created. (#823) * Append to file if output file is already created. * Update file permission to 0644 to 0600 * Only append if resume flag is passed otherwise create new * Remove asn number failing test case --- runner/runner.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index b2f06e4..79fc117 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -591,7 +591,11 @@ func (r *Runner) RunEnumeration() { var f *os.File if r.options.Output != "" { var err error - f, err = os.Create(r.options.Output) + if r.options.Resume { + f, err = os.OpenFile(r.options.Output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + } else { + f, err = os.Create(r.options.Output) + } if err != nil { gologger.Fatal().Msgf("Could not create output file '%s': %s\n", r.options.Output, err) } From 1bb544863361520165f1ff68eefe45a03e231036 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 29 Oct 2022 09:17:26 +0000 Subject: [PATCH 296/355] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.1 to 0.1.2. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.1...v0.1.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3e51c64..8c5fc0a 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.3 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.1.1 + github.com/projectdiscovery/goflags v0.1.2 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 diff --git a/go.sum b/go.sum index 2e5a898..5957ce9 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytl github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= -github.com/projectdiscovery/goflags v0.1.1 h1:AEtT14D9OC10HWyZwDQaSLjuK8ZKoBrSYlsLItvMKZI= -github.com/projectdiscovery/goflags v0.1.1/go.mod h1:/YBPA+1igSkQbwD7a91o0HUIwMDlsmQDRZL2oSYSyEQ= +github.com/projectdiscovery/goflags v0.1.2 h1:osaVfg0VOMhEUvUCgAJLizwu3kFH1YeO2JiJYVJKLvg= +github.com/projectdiscovery/goflags v0.1.2/go.mod h1:hI/9sZpiIt+DE6RkNPTo98/u5OhR3EnoH5ozidT0HtY= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -313,8 +313,6 @@ github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655/go.mod github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= -github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa h1:XTRcow+zyZUYxOlLsSp1AdM4Jl+i/OpPbOZPsfU/IAI= -github.com/projectdiscovery/ratelimit v0.0.0-20221004232058-7b82379157fa/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= github.com/projectdiscovery/ratelimit v0.0.1 h1:GnCfbKmkLdDLXT3QS4KS0zCsuDGkoRQE0YDbTqzQmS8= github.com/projectdiscovery/ratelimit v0.0.1/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= From 5e99c41c13a65dc146e751239f77771f3032693c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 10:10:40 +0000 Subject: [PATCH 297/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.66 to 0.0.67. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.66...v0.0.67) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8c5fc0a..a9ce8bb 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 github.com/projectdiscovery/stringsutil v0.0.2 github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.66 + github.com/projectdiscovery/wappalyzergo v0.0.67 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 5957ce9..1f40569 100644 --- a/go.sum +++ b/go.sum @@ -340,8 +340,8 @@ github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjb github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= -github.com/projectdiscovery/wappalyzergo v0.0.66 h1:+egr7a8VRNtSeqDDap3IYPW2rH+sCpB9Irwp94ipA7E= -github.com/projectdiscovery/wappalyzergo v0.0.66/go.mod h1:9aSADdt5z/pw9LFZF7Q8RrLnkyqZl1H4Ezivi8Td7l0= +github.com/projectdiscovery/wappalyzergo v0.0.67 h1:4ckerDUmT/OiU+O3XZ0ZGGEJ4DDPVAlsYtXFIzLtfjQ= +github.com/projectdiscovery/wappalyzergo v0.0.67/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= From f209788220ade10fc873bd2ebdfaaa7c2022c088 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Nov 2022 10:17:53 +0000 Subject: [PATCH 298/355] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.2 to 0.1.3. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.2...v0.1.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a9ce8bb..21a7540 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.3 github.com/projectdiscovery/fileutil v0.0.3 github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.1.2 + github.com/projectdiscovery/goflags v0.1.3 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2 github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 diff --git a/go.sum b/go.sum index 1f40569..c6897a3 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytl github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= -github.com/projectdiscovery/goflags v0.1.2 h1:osaVfg0VOMhEUvUCgAJLizwu3kFH1YeO2JiJYVJKLvg= -github.com/projectdiscovery/goflags v0.1.2/go.mod h1:hI/9sZpiIt+DE6RkNPTo98/u5OhR3EnoH5ozidT0HtY= +github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M= +github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= From e40367d9040b29c3bc3ece1cea356352e9b03dfd Mon Sep 17 00:00:00 2001 From: foxcores <116920930+foxcores@users.noreply.github.com> Date: Wed, 2 Nov 2022 09:33:19 +0100 Subject: [PATCH 299/355] Updating httpx as a library integration test (#847) * Update library.go Go 1.18 + Some missing tests #826 * using standard http lib Co-authored-by: Mzack9999 --- cmd/integration-test/library.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index f20e4bf..a81432a 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -1,6 +1,7 @@ package main import ( + "net/http" "os" "github.com/projectdiscovery/httpx/internal/testutils" @@ -22,9 +23,14 @@ func (h *httpxLibrary) Execute() error { } defer os.RemoveAll(testFile) + var got string + options := runner.Options{ - Methods: "GET", + Methods: http.MethodGet, InputFile: testFile, + OnResult: func(r runner.Result) { + got = r.URL + }, } if err := options.ValidateOptions(); err != nil { return err @@ -37,5 +43,12 @@ func (h *httpxLibrary) Execute() error { defer httpxRunner.Close() httpxRunner.RunEnumeration() + + expected := "https://scanme.sh:443" + + if got != expected { + return errIncorrectResult(expected, got) + } + return nil } From baa0fa0d2944fe89857b7039bd92d54b6e4b1d11 Mon Sep 17 00:00:00 2001 From: foxcores <116920930+foxcores@users.noreply.github.com> Date: Sat, 5 Nov 2022 21:37:53 +0100 Subject: [PATCH 300/355] Fix issue #849 --- cmd/integration-test/http.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 5637961..654d50c 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -83,7 +83,7 @@ func (h *standardHttpGet) Execute() error { } if h.expectedOutput != "" && !strings.EqualFold(results[0], h.expectedOutput) { - return errIncorrectResult(results[0], h.expectedOutput) + return errIncorrectResult(h.expectedOutput, results[0]) } return nil @@ -115,7 +115,7 @@ func (h *issue276) Execute() error { // title expected := ts.URL + "/redirect" + " [302] [Object moved]" if !strings.EqualFold(results[0], expected) { - return errIncorrectResult(results[0], expected) + return errIncorrectResult(expected, results[0]) } return nil } @@ -146,7 +146,7 @@ func (h *issue277) Execute() error { // title expected := ts.URL + uripath if !strings.EqualFold(results[0], expected) { - return errIncorrectResult(results[0], expected) + return errIncorrectResult(expected, results[0]) } return nil } @@ -175,7 +175,7 @@ func (h *issue303) Execute() error { // full url with port expected := ts.URL if !strings.EqualFold(results[0], expected) { - return errIncorrectResult(results[0], expected) + return errIncorrectResult(expected, results[0]) } return nil } @@ -249,7 +249,7 @@ func (h *issue414) Execute() error { } expected := ts.URL + uripath if !strings.EqualFold(results[0], expected) { - return errIncorrectResult(results[0], expected) + return errIncorrectResult(expected, results[0]) } return nil } From 686a2cced80c5157eb48ad65b09983f8d76c9e96 Mon Sep 17 00:00:00 2001 From: vrenzolaverace Date: Sun, 6 Nov 2022 21:22:47 +0100 Subject: [PATCH 301/355] Update helpers libraries #850 (#851) Use utils helpers libraries (#850) --- cmd/functional-test/main.go | 2 +- common/customheader/customheader.go | 3 ++- common/customports/customport.go | 2 +- common/fileutil/fileutil.go | 2 +- common/httputilz/httputilz.go | 2 +- common/httpx/csp.go | 2 +- common/httpx/encodings.go | 2 +- common/httpx/httpx.go | 4 ++-- common/httpx/response.go | 2 +- common/httpx/title.go | 2 +- common/stringz/stringz.go | 2 +- go.mod | 13 +++++++------ go.sum | 18 ++++++++++++++++++ runner/healthcheck.go | 2 +- runner/options.go | 2 +- runner/runner.go | 14 +++++++------- runner/types.go | 2 +- 17 files changed, 48 insertions(+), 28 deletions(-) diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go index 52a3db3..3fd4242 100644 --- a/cmd/functional-test/main.go +++ b/cmd/functional-test/main.go @@ -22,7 +22,7 @@ var ( mainHttpxBinary = flag.String("main", "", "Main Branch Httpx Binary") devHttpxBinary = flag.String("dev", "", "Dev Branch Httpx Binary") - testcases = flag.String("testcases", "", "Test cases file for Httpx functional tests") + testcases = flag.String("testcases", "", "Test cases file for Httpx functional tests") ) func main() { diff --git a/common/customheader/customheader.go b/common/customheader/customheader.go index a739b00..93eea0b 100644 --- a/common/customheader/customheader.go +++ b/common/customheader/customheader.go @@ -2,7 +2,8 @@ package customheader import ( "strings" - "github.com/projectdiscovery/stringsutil" + + stringsutil "github.com/projectdiscovery/utils/strings" ) // CustomHeaders valid for all requests diff --git a/common/customports/customport.go b/common/customports/customport.go index 60df7e0..9a08def 100644 --- a/common/customports/customport.go +++ b/common/customports/customport.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/pkg/errors" - "github.com/projectdiscovery/sliceutil" + sliceutil "github.com/projectdiscovery/utils/slice" "github.com/projectdiscovery/httpx/common/httpx" ) diff --git a/common/fileutil/fileutil.go b/common/fileutil/fileutil.go index 76213cb..8efe89f 100644 --- a/common/fileutil/fileutil.go +++ b/common/fileutil/fileutil.go @@ -9,8 +9,8 @@ import ( "path/filepath" "regexp" - "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/httpx/common/stringz" + fileutil "github.com/projectdiscovery/utils/file" ) // HasStdin determines if the user has piped input diff --git a/common/httputilz/httputilz.go b/common/httputilz/httputilz.go index 1aaa6fc..94bafa8 100644 --- a/common/httputilz/httputilz.go +++ b/common/httputilz/httputilz.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/urlutil" + urlutil "github.com/projectdiscovery/utils/url" ) const ( diff --git a/common/httpx/csp.go b/common/httpx/csp.go index e1c4831..c356cfc 100644 --- a/common/httpx/csp.go +++ b/common/httpx/csp.go @@ -6,7 +6,7 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/projectdiscovery/httpx/common/slice" - "github.com/projectdiscovery/stringsutil" + stringsutil "github.com/projectdiscovery/utils/strings" ) // CSPHeaders is an incomplete list of most common CSP headers diff --git a/common/httpx/encodings.go b/common/httpx/encodings.go index 9c5114b..ab119b8 100644 --- a/common/httpx/encodings.go +++ b/common/httpx/encodings.go @@ -6,7 +6,7 @@ import ( "net/http" "strings" - "github.com/projectdiscovery/stringsutil" + stringsutil "github.com/projectdiscovery/utils/strings" "golang.org/x/text/encoding/korean" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/encoding/traditionalchinese" diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 81e0990..300a856 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -16,10 +16,10 @@ import ( "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" - pdhttputil "github.com/projectdiscovery/httputil" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/stringsutil" + pdhttputil "github.com/projectdiscovery/utils/http" + stringsutil "github.com/projectdiscovery/utils/strings" "golang.org/x/net/context" "golang.org/x/net/http2" ) diff --git a/common/httpx/response.go b/common/httpx/response.go index fa561f7..bf0e15f 100644 --- a/common/httpx/response.go +++ b/common/httpx/response.go @@ -4,8 +4,8 @@ import ( "strings" "time" - "github.com/projectdiscovery/httputil" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" + httputil "github.com/projectdiscovery/utils/http" ) // Response contains the response to a server diff --git a/common/httpx/title.go b/common/httpx/title.go index 4f799a8..efcd477 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -7,7 +7,7 @@ import ( "regexp" "strings" - "github.com/projectdiscovery/stringsutil" + stringsutil "github.com/projectdiscovery/utils/strings" "golang.org/x/net/html" ) diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index 6c73201..8d8a140 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - "github.com/projectdiscovery/urlutil" + urlutil "github.com/projectdiscovery/utils/url" "github.com/spaolacci/murmur3" ) diff --git a/go.mod b/go.mod index 21a7540..994a5b8 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/projectdiscovery/tlsx v0.0.9 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.8.0 - golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 + golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 ) require ( @@ -95,18 +95,19 @@ require ( github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.15 // indirect + github.com/projectdiscovery/utils v0.0.1 // indirect github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect - github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 // indirect + github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect - github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c // indirect - golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/tools v0.1.12 // indirect + github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be // indirect + golang.org/x/crypto v0.1.0 // indirect + golang.org/x/mod v0.6.0 // indirect + golang.org/x/tools v0.2.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index c6897a3..a998d7b 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -340,6 +341,8 @@ github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjb github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= +github.com/projectdiscovery/utils v0.0.1 h1:8jxai1EF3z/lpHeUCQEyNvEzs2Y53bzsMzEhoFjjwo0= +github.com/projectdiscovery/utils v0.0.1/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI= github.com/projectdiscovery/wappalyzergo v0.0.67 h1:4ckerDUmT/OiU+O3XZ0ZGGEJ4DDPVAlsYtXFIzLtfjQ= github.com/projectdiscovery/wappalyzergo v0.0.67/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -400,9 +403,13 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 h1:vMJBP3PQViZsF6cOINtvyMC8ptpLsyJ4EwyFnzuWNxc= github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= +github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 h1:oRCu5zb6sklsDvy5sOz3dFqGg5vAEYBBD2MAYhNThCQ= +github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= +github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -424,6 +431,8 @@ github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54t github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw= github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c h1:ufDm/IlBYZYLuiqvQuhpTKwrcAS2OlXEzWbDvTVGbSQ= github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c/go.mod h1:egdRkzUylATvPkWMpebZbXhv0FMEMJGX/ur0D3Csk2s= +github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be h1:M5QjuCbUeNZsup53dlJkI/cx6pVdnDOPzyy+XppoowY= +github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= @@ -444,11 +453,16 @@ golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 h1:vJ2V3lFLg+bBhgroYuRfyN583UzVveQmIXjc8T/y3to= golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 h1:17k44ji3KFYG94XS5QEFC8pyuOlMh3IoR+vkmTZmJJs= golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -462,6 +476,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -572,6 +588,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/runner/healthcheck.go b/runner/healthcheck.go index ddb3937..c48490f 100644 --- a/runner/healthcheck.go +++ b/runner/healthcheck.go @@ -6,8 +6,8 @@ import ( "runtime" "strings" - "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goflags" + fileutil "github.com/projectdiscovery/utils/file" ) func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { diff --git a/runner/options.go b/runner/options.go index 86971c7..4001b93 100644 --- a/runner/options.go +++ b/runner/options.go @@ -11,7 +11,6 @@ import ( "golang.org/x/exp/maps" "github.com/projectdiscovery/cdncheck" - "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" @@ -24,6 +23,7 @@ import ( fileutilz "github.com/projectdiscovery/httpx/common/fileutil" "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/httpx/common/stringz" + fileutil "github.com/projectdiscovery/utils/file" ) const ( diff --git a/runner/runner.go b/runner/runner.go index 79fc117..5347c3e 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -29,7 +29,7 @@ import ( "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" "github.com/projectdiscovery/mapcidr/asn" - "github.com/projectdiscovery/mapsutil" + mapsutil "github.com/projectdiscovery/utils/maps" "github.com/bluele/gcache" "github.com/logrusorgru/aurora" @@ -39,28 +39,28 @@ import ( "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/httpx/common/hashes" "github.com/projectdiscovery/retryablehttp-go" - "github.com/projectdiscovery/sliceutil" - "github.com/projectdiscovery/stringsutil" - "github.com/projectdiscovery/urlutil" + sliceutil "github.com/projectdiscovery/utils/slice" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" "github.com/projectdiscovery/ratelimit" "github.com/remeh/sizedwaitgroup" // automatic fd max increase if running as root _ "github.com/projectdiscovery/fdmax/autofdmax" - "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/hmap/store/hybrid" - pdhttputil "github.com/projectdiscovery/httputil" customport "github.com/projectdiscovery/httpx/common/customports" fileutilz "github.com/projectdiscovery/httpx/common/fileutil" "github.com/projectdiscovery/httpx/common/httputilz" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/httpx/common/stringz" - "github.com/projectdiscovery/iputil" "github.com/projectdiscovery/mapcidr" "github.com/projectdiscovery/rawhttp" + fileutil "github.com/projectdiscovery/utils/file" + pdhttputil "github.com/projectdiscovery/utils/http" + iputil "github.com/projectdiscovery/utils/ip" wappalyzer "github.com/projectdiscovery/wappalyzergo" ) diff --git a/runner/types.go b/runner/types.go index 77e5b96..c94a6c6 100644 --- a/runner/types.go +++ b/runner/types.go @@ -8,8 +8,8 @@ import ( "github.com/bxcodec/faker/v4/pkg/options" "github.com/mitchellh/mapstructure" "github.com/projectdiscovery/httpx/common/httpx" - "github.com/projectdiscovery/mapsutil" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" + mapsutil "github.com/projectdiscovery/utils/maps" ) type AsnResponse struct { From 777e7d58028fccf3304ba44cc80755b05271e8c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 10:01:06 +0000 Subject: [PATCH 302/355] chore(deps): bump golang from 1.19.2-alpine to 1.19.3-alpine Bumps golang from 1.19.2-alpine to 1.19.3-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4c0ffbb..bf74554 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.2-alpine AS builder +FROM golang:1.19.3-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From 4d5910c72ca710ecb571055eeeb398eda58c42f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Nov 2022 10:04:52 +0000 Subject: [PATCH 303/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.67 to 0.0.68. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.67...v0.0.68) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 15 ++++++--------- go.sum | 22 ++-------------------- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 994a5b8..3dde6fc 100644 --- a/go.mod +++ b/go.mod @@ -18,20 +18,18 @@ require ( github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/fastdialer v0.0.17 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.3 + github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.3 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2 - github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 - github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 + github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 // indirect github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d - github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 - github.com/projectdiscovery/stringsutil v0.0.2 - github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 - github.com/projectdiscovery/wappalyzergo v0.0.67 + github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 // indirect + github.com/projectdiscovery/stringsutil v0.0.2 // indirect + github.com/projectdiscovery/wappalyzergo v0.0.68 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect @@ -52,9 +50,9 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/asnmap v0.0.1 github.com/projectdiscovery/dsl v0.0.3 - github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655 github.com/projectdiscovery/ratelimit v0.0.1 github.com/projectdiscovery/tlsx v0.0.9 + github.com/projectdiscovery/utils v0.0.1 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 @@ -95,7 +93,6 @@ require ( github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.15 // indirect - github.com/projectdiscovery/utils v0.0.1 // indirect github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect diff --git a/go.sum b/go.sum index a998d7b..6930462 100644 --- a/go.sum +++ b/go.sum @@ -298,8 +298,6 @@ github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8q github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/hmap v0.0.2 h1:fe3k0b6tj95mn9a1phD3JXvAAOmmWAh/Upg1Bf0Tfos= github.com/projectdiscovery/hmap v0.0.2/go.mod h1:YU3TeNTDmLW2dtb4LvuEtDsPTsQ06XMnmOeD3KOuU6c= -github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 h1:GzruqQhb+sj1rEuHRFLhWX8gH/tJ+sj1udRjOy9VCJo= -github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 h1:HRqev12wKvcwK1fe4pSlMfQdPHo9LfTxuFeRN4f3tS4= @@ -309,8 +307,6 @@ github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJ github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= -github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655 h1:6FTqSAohPHeBnzTypby8Gk88zv03PfrRMKQtRCFELrU= -github.com/projectdiscovery/mapsutil v0.0.2-0.20221020054123-d68d6f24a655/go.mod h1:NIgEkt1yYqHWYi1Razavwa1J2hinHfSwD287ZtKBQjU= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= @@ -329,7 +325,6 @@ github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26 github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI= -github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= @@ -339,12 +334,10 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= -github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= -github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= github.com/projectdiscovery/utils v0.0.1 h1:8jxai1EF3z/lpHeUCQEyNvEzs2Y53bzsMzEhoFjjwo0= github.com/projectdiscovery/utils v0.0.1/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI= -github.com/projectdiscovery/wappalyzergo v0.0.67 h1:4ckerDUmT/OiU+O3XZ0ZGGEJ4DDPVAlsYtXFIzLtfjQ= -github.com/projectdiscovery/wappalyzergo v0.0.67/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/wappalyzergo v0.0.68 h1:QUo/njWarb8GxAf8b8gCKDEuUXTKSPQMLCY4dq/wqfM= +github.com/projectdiscovery/wappalyzergo v0.0.68/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= @@ -405,8 +398,6 @@ github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPU github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= -github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236 h1:vMJBP3PQViZsF6cOINtvyMC8ptpLsyJ4EwyFnzuWNxc= -github.com/weppos/publicsuffix-go v0.15.1-0.20220329081811-9a40b608a236/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 h1:oRCu5zb6sklsDvy5sOz3dFqGg5vAEYBBD2MAYhNThCQ= github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= @@ -429,8 +420,6 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhu github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 h1:17HHAgFKlLcZsDOjBOUrd5hDihb1ggf+1a5dTbkgkIY= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw= -github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c h1:ufDm/IlBYZYLuiqvQuhpTKwrcAS2OlXEzWbDvTVGbSQ= -github.com/zmap/zcrypto v0.0.0-20220605182715-4dfcec6e9a8c/go.mod h1:egdRkzUylATvPkWMpebZbXhv0FMEMJGX/ur0D3Csk2s= github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be h1:M5QjuCbUeNZsup53dlJkI/cx6pVdnDOPzyy+XppoowY= github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -451,16 +440,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 h1:vJ2V3lFLg+bBhgroYuRfyN583UzVveQmIXjc8T/y3to= -golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 h1:17k44ji3KFYG94XS5QEFC8pyuOlMh3IoR+vkmTZmJJs= -golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20221019170559-20944726eadf/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -474,7 +458,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= @@ -586,7 +569,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= From 3bc5a7dea4c3135b61d713694c6df67ee0707de7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Nov 2022 16:07:56 +0530 Subject: [PATCH 304/355] chore(deps): bump github.com/projectdiscovery/utils from 0.0.1 to 0.0.2 (#857) Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.1 to 0.0.2. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.1...v0.0.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3dde6fc..587c326 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/ratelimit v0.0.1 github.com/projectdiscovery/tlsx v0.0.9 - github.com/projectdiscovery/utils v0.0.1 + github.com/projectdiscovery/utils v0.0.2 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 diff --git a/go.sum b/go.sum index 6930462..93cc1cf 100644 --- a/go.sum +++ b/go.sum @@ -334,8 +334,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= -github.com/projectdiscovery/utils v0.0.1 h1:8jxai1EF3z/lpHeUCQEyNvEzs2Y53bzsMzEhoFjjwo0= -github.com/projectdiscovery/utils v0.0.1/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI= +github.com/projectdiscovery/utils v0.0.2 h1:lif4OYBqd8jCf0glRBfSs2lT4nMtNjHjeysRw8HIW8M= +github.com/projectdiscovery/utils v0.0.2/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI= github.com/projectdiscovery/wappalyzergo v0.0.68 h1:QUo/njWarb8GxAf8b8gCKDEuUXTKSPQMLCY4dq/wqfM= github.com/projectdiscovery/wappalyzergo v0.0.68/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= From 9d4f6afa8a5cb1b1d4e2be5b1bc1eb37adf53985 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 9 Nov 2022 16:32:56 +0100 Subject: [PATCH 305/355] Fixing content length bytes count --- common/httpx/httpx.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 300a856..56041df 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -9,7 +9,6 @@ import ( "strconv" "strings" "time" - "unicode/utf8" "github.com/corpix/uarand" "github.com/microcosm-cc/bluemonday" @@ -232,14 +231,14 @@ get_response: // if content length is not defined if resp.ContentLength <= 0 { // check if it's in the header and convert to int - if contentLength, ok := resp.Headers["Content-Length"]; ok { - contentLengthInt, _ := strconv.Atoi(strings.Join(contentLength, "")) + if contentLength, ok := resp.Headers["Content-Length"]; ok && len(contentLength) > 0 { + contentLengthInt, _ := strconv.Atoi(contentLength[0]) resp.ContentLength = contentLengthInt } // if we have a body, then use the number of bytes in the body if the length is still zero - if resp.ContentLength <= 0 && len(respbodystr) > 0 { - resp.ContentLength = utf8.RuneCountInString(respbodystr) + if resp.ContentLength <= 0 && len(respbody) > 0 { + resp.ContentLength = len(respbody) } } From f327e1cc777ef6fd38b1b3c3ade7b2d92dd62d7a Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Wed, 9 Nov 2022 17:02:18 +0100 Subject: [PATCH 306/355] adding generic CL test cases --- common/httpx/httpx_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 common/httpx/httpx_test.go diff --git a/common/httpx/httpx_test.go b/common/httpx/httpx_test.go new file mode 100644 index 0000000..5f039b0 --- /dev/null +++ b/common/httpx/httpx_test.go @@ -0,0 +1,30 @@ +package httpx + +import ( + "net/http" + "testing" + + "github.com/projectdiscovery/retryablehttp-go" + "github.com/stretchr/testify/require" +) + +func TestDo(t *testing.T) { + ht, err := New(&DefaultOptions) + require.Nil(t, err) + + t.Run("content-length in header", func(t *testing.T) { + req, err := retryablehttp.NewRequest(http.MethodGet, "https://scanme.sh", nil) + require.Nil(t, err) + resp, err := ht.Do(req, UnsafeOptions{}) + require.Nil(t, err) + require.Equal(t, 2, resp.ContentLength) + }) + + t.Run("content-length with binary body", func(t *testing.T) { + req, err := retryablehttp.NewRequest(http.MethodGet, "https://www.w3schools.com/images/favicon.ico", nil) + require.Nil(t, err) + resp, err := ht.Do(req, UnsafeOptions{}) + require.Nil(t, err) + require.Equal(t, 318, resp.ContentLength) + }) +} From 875ded36c50ecc264ca86e7897ebd9bc16770a11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Nov 2022 20:06:49 +0530 Subject: [PATCH 307/355] chore(deps): bump golang.org/x/net from 0.1.0 to 0.2.0 (#858) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.1.0 to 0.2.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.1.0...v0.2.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 587c326..90837c5 100644 --- a/go.mod +++ b/go.mod @@ -35,8 +35,8 @@ require ( github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect - golang.org/x/net v0.1.0 - golang.org/x/sys v0.1.0 // indirect + golang.org/x/net v0.2.0 + golang.org/x/sys v0.2.0 // indirect golang.org/x/text v0.4.0 ) diff --git a/go.sum b/go.sum index 93cc1cf..b9a90d2 100644 --- a/go.sum +++ b/go.sum @@ -491,8 +491,9 @@ golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -540,8 +541,9 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 32a5bab2e8b2b3a8546f9ff7a8e50add03e5794c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 10:01:12 +0000 Subject: [PATCH 308/355] chore(deps): bump golangci/golangci-lint-action from 3.3.0 to 3.3.1 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.3.0...v3.3.1) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index d6921ef..40cef36 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -16,7 +16,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.3.0 + uses: golangci/golangci-lint-action@v3.3.1 with: version: latest args: --timeout 5m From 93de239700e031dce7990042e8709d8a37024b70 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 10:01:15 +0000 Subject: [PATCH 309/355] chore(deps): bump alpine from 3.16.2 to 3.16.3 Bumps alpine from 3.16.2 to 3.16.3. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bf74554..2088447 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.19.3-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.16.2 +FROM alpine:3.16.3 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From 2cd8a1672e71ad1964b1d31d44d57e7556426043 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 10:05:12 +0000 Subject: [PATCH 310/355] chore(deps): bump github.com/projectdiscovery/utils from 0.0.2 to 0.0.3 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.2 to 0.0.3. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.2...v0.0.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 90837c5..f80c93f 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect - go.uber.org/atomic v1.9.0 // indirect + go.uber.org/atomic v1.10.0 // indirect golang.org/x/net v0.2.0 golang.org/x/sys v0.2.0 // indirect golang.org/x/text v0.4.0 @@ -52,7 +52,7 @@ require ( github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/ratelimit v0.0.1 github.com/projectdiscovery/tlsx v0.0.9 - github.com/projectdiscovery/utils v0.0.2 + github.com/projectdiscovery/utils v0.0.3 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.8.0 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 diff --git a/go.sum b/go.sum index b9a90d2..b62bf15 100644 --- a/go.sum +++ b/go.sum @@ -334,8 +334,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= -github.com/projectdiscovery/utils v0.0.2 h1:lif4OYBqd8jCf0glRBfSs2lT4nMtNjHjeysRw8HIW8M= -github.com/projectdiscovery/utils v0.0.2/go.mod h1:PEYYkpCedmtydQRUvfWLPw0VRWpaFms4GFqNAziBANI= +github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0dudpcGfM= +github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= github.com/projectdiscovery/wappalyzergo v0.0.68 h1:QUo/njWarb8GxAf8b8gCKDEuUXTKSPQMLCY4dq/wqfM= github.com/projectdiscovery/wappalyzergo v0.0.68/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -426,8 +426,9 @@ go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= From 7acf7f3607582c00a94ba11b8a23bb6b67e5d27f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Nov 2022 12:08:53 +0530 Subject: [PATCH 311/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.68 to 0.0.69 (#864) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.68 to 0.0.69. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.68...v0.0.69) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f80c93f..8aa0200 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/projectdiscovery/wappalyzergo v0.0.68 + github.com/projectdiscovery/wappalyzergo v0.0.69 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index b62bf15..60e22bb 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjb github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0dudpcGfM= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= -github.com/projectdiscovery/wappalyzergo v0.0.68 h1:QUo/njWarb8GxAf8b8gCKDEuUXTKSPQMLCY4dq/wqfM= -github.com/projectdiscovery/wappalyzergo v0.0.68/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/wappalyzergo v0.0.69 h1:x5etq0b+bxQg3dhbSfOEYoV5ODl6M+S99KxyXLdkuFY= +github.com/projectdiscovery/wappalyzergo v0.0.69/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= From f06133f03b787d34c30a33a466ac0ea4f3d3689c Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Tue, 15 Nov 2022 12:23:45 +0530 Subject: [PATCH 312/355] Add delay between making httpx requests. (#853) * Add delay between making httpx requests. - sleep before creating goroutine to make http request - eg. ``` cat urls.txt | httpx -delay 2s ``` ``` cat urls.txt | httpx -delay 2s -port 80,443,8443 ``` * Update delay help message * Use goflags DurationVar to accept delay * replace string with duration for delay --- README.md | 1 + runner/options.go | 3 +++ runner/runner.go | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 06c96ee..a802fb1 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,7 @@ OPTIMIZATIONS: -ec, -exclude-cdn skip full port scans for CDNs (only checks for 80,443) -retries int number of retries -timeout int timeout in seconds (default 5) + -delay duration duration to wait between each connection per thread (eg: 200ms, 1s) -rsts, -response-size-to-save int max response size to save in bytes (default 2147483647) -rstr, -response-size-to-read int max response size to read in bytes (default 2147483647) ``` diff --git a/runner/options.go b/runner/options.go index 4001b93..81b5996 100644 --- a/runner/options.go +++ b/runner/options.go @@ -6,6 +6,7 @@ import ( "os" "regexp" "strings" + "time" "github.com/pkg/errors" "golang.org/x/exp/maps" @@ -164,6 +165,7 @@ type Options struct { Retries int Threads int Timeout int + Delay time.Duration filterRegex *regexp.Regexp matchRegex *regexp.Regexp VHost bool @@ -396,6 +398,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.ExcludeCDN, "exclude-cdn", "ec", false, "skip full port scans for CDNs (only checks for 80,443)"), flagSet.IntVar(&options.Retries, "retries", 0, "number of retries"), flagSet.IntVar(&options.Timeout, "timeout", 5, "timeout in seconds"), + flagSet.DurationVar(&options.Delay, "delay", -1, "duration between each http request (eg: 200ms, 1s)"), flagSet.IntVarP(&options.MaxResponseBodySizeToSave, "response-size-to-save", "rsts", math.MaxInt32, "max response size to save in bytes"), flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"), ) diff --git a/runner/runner.go b/runner/runner.go index 5347c3e..7a429c2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -854,6 +854,8 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. if len(customport.Ports) == 0 { for _, method := range scanopts.Methods { for _, prot := range protocols { + // sleep for delay time + time.Sleep(r.options.Delay) wg.Add() go func(target httpx.Target, method, protocol string) { defer wg.Done() @@ -892,6 +894,8 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. } for _, wantedProtocol := range wantedProtocols { for _, method := range scanopts.Methods { + // sleep for delay time + time.Sleep(r.options.Delay) wg.Add() go func(port int, target httpx.Target, method, protocol string) { defer wg.Done() From 1f409072b4927f4a2398ab9109b2045c2f67e592 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 16 Nov 2022 23:22:27 +0200 Subject: [PATCH 313/355] Readme update (#867) --- README.md | 49 ++++++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index a802fb1..461ddaf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- httpx + httpx

@@ -24,12 +24,12 @@

-httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library. It is designed to maintain result reliability with an increased number of threads. +`httpx` is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library. It is designed to maintain result reliability with an increased number of threads. # Features

- httpx + httpx

@@ -40,7 +40,7 @@ httpx is a fast and multi-purpose HTTP toolkit that allows running multiple prob - Supports hosts, URLs and CIDR as input. - Handles edge cases doing retries, backoffs etc for handling WAFs. -### Supported probes:- +### Supported probes | Probes | Default check | Probes | Default check | |-----------------|---------------|----------------|---------------| @@ -62,7 +62,7 @@ httpx is a fast and multi-purpose HTTP toolkit that allows running multiple prob # Installation Instructions -httpx requires **go1.17** to install successfully. Run the following command to get the repo - +`httpx` requires **go1.18** to install successfully. Run the following command to get the repo: ```sh go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest @@ -204,7 +204,7 @@ OPTIMIZATIONS: -ec, -exclude-cdn skip full port scans for CDNs (only checks for 80,443) -retries int number of retries -timeout int timeout in seconds (default 5) - -delay duration duration to wait between each connection per thread (eg: 200ms, 1s) + -delay duration duration between each http request (eg: 200ms, 1s) (default -1ns) -rsts, -response-size-to-save int max response size to save in bytes (default 2147483647) -rstr, -response-size-to-read int max response size to read in bytes (default 2147483647) ``` @@ -242,7 +242,7 @@ https://support.hackerone.com ### File Input -This will run the tool with the `probe` flag against all of the hosts in **hosts.txt** and return URLs with probed status. +This will run the tool with the `-probe` flag against all the hosts in **hosts.txt** and return URLs with probed status. ```console httpx -list hosts.txt -silent -probe @@ -468,31 +468,30 @@ https://docs.hackerone.com https://support.hackerone.com ``` -### Using httpx as a library -`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to close at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it: +### Using `httpx` as a library +`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it: ```go package main import ( "log" - "os" + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/httpx/runner" ) func main() { - inputFile := "test.txt" - err := os.WriteFile(inputFile, []byte("scanme.sh"), 0644) - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(inputFile) + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional) options := runner.Options{ - Methods: "GET", - InputFile: inputFile, + Methods: "GET", + InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"}, + //InputFile: "./targetDomains.txt", // path to file containing the target domains list } + if err := options.ValidateOptions(); err != nil { log.Fatal(err) } @@ -510,17 +509,17 @@ func main() { # Notes -- As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable. -- For printing both HTTP/HTTPS results, `no-fallback` flag can be used. +- As default, `httpx` checks for **HTTPS** probe and fall-back to **HTTP** only if **HTTPS** is not reachable. +- The `-no-fallback` flag can be used to display both **HTTP** and **HTTPS** results - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` -- `favicon`,`vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes. -- Unique flags should be used for specific use cases instead of running them as default with other probes. -- When using `json` flag, all the information (default probes) included in the JSON output. -- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (eg **udp:127.0.0.1:53**) +- The following flags should be used for specific use cases instead of running them as default with other probes: + * `-favicon`,`-vhost`, `-http2`, `-pipeline`, `-ports`, `-csp-probe`, `-tls-probe`, `-path` +- When using the `-json` flag, all the default probe results are included in the JSON output. +- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) - Invalid custom resolvers/files are ignored. # Acknowledgement -httpx is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/httpx/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow: +`httpx` is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/httpx/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow: Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomnom/httprobe) work :heart: From 6451bf79254257d593c157e881723d5e94d010b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 20 Nov 2022 00:31:47 +0530 Subject: [PATCH 314/355] chore(deps): bump github.com/projectdiscovery/fdmax from 0.0.3 to 0.0.4 (#869) Bumps [github.com/projectdiscovery/fdmax](https://github.com/projectdiscovery/fdmax) from 0.0.3 to 0.0.4. - [Release notes](https://github.com/projectdiscovery/fdmax/releases) - [Commits](https://github.com/projectdiscovery/fdmax/compare/v0.0.3...v0.0.4) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fdmax dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8aa0200..80c3ed3 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/fastdialer v0.0.17 - github.com/projectdiscovery/fdmax v0.0.3 + github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.3 diff --git a/go.sum b/go.sum index 60e22bb..ae1be2b 100644 --- a/go.sum +++ b/go.sum @@ -278,8 +278,8 @@ github.com/projectdiscovery/dsl v0.0.3 h1:oWlZZaSADqoyfJdUHWqAzpB65NpvLukZQGFv1u github.com/projectdiscovery/dsl v0.0.3/go.mod h1:ST66slxtp7fsFqTOq3k6+1TDV7RH1Moz3sPBqUGn4Fg= github.com/projectdiscovery/fastdialer v0.0.17 h1:wT7LinZAcyEzkfhJLhWNQrA9m79+hTd6CJ7aWTJfXjk= github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= -github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= -github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= +github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= +github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= @@ -530,7 +530,6 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 69f3eec5e1aac3cdb0a361c9cc6ecd1f35a55f52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 20 Nov 2022 00:32:04 +0530 Subject: [PATCH 315/355] chore(deps): bump github.com/projectdiscovery/clistats from 0.0.8 to 0.0.9 (#868) chore(deps): bump github.com/projectdiscovery/clistats Bumps [github.com/projectdiscovery/clistats](https://github.com/projectdiscovery/clistats) from 0.0.8 to 0.0.9. - [Release notes](https://github.com/projectdiscovery/clistats/releases) - [Commits](https://github.com/projectdiscovery/clistats/compare/v0.0.8...v0.0.9) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/clistats dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 80c3ed3..f98b168 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/miekg/dns v1.1.50 // indirect github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 - github.com/projectdiscovery/clistats v0.0.8 + github.com/projectdiscovery/clistats v0.0.9 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect github.com/projectdiscovery/fastdialer v0.0.17 github.com/projectdiscovery/fdmax v0.0.4 diff --git a/go.sum b/go.sum index ae1be2b..f47128f 100644 --- a/go.sum +++ b/go.sum @@ -270,8 +270,8 @@ github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= -github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= -github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= +github.com/projectdiscovery/clistats v0.0.9 h1:8sA17+2qP+cTnY7LaaGURJW5stSM8UwQiygwHQjxHx4= +github.com/projectdiscovery/clistats v0.0.9/go.mod h1:7F1RdeGAoLf05rhsZesL0+qoXJpOA/vxuRj2JRIAzU4= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/dsl v0.0.3 h1:oWlZZaSADqoyfJdUHWqAzpB65NpvLukZQGFv1uTtU3g= @@ -438,7 +438,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= @@ -528,7 +527,6 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 9771cc106dba5435a6acfe0c3385c87618cae730 Mon Sep 17 00:00:00 2001 From: ndoell Date: Sun, 20 Nov 2022 00:44:01 -0500 Subject: [PATCH 316/355] Fix for failing build on gcc error. (#871) https://github.com/projectdiscovery/httpx/issues/870 Co-authored-by: Sandeep Singh Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2088447..f2150e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM golang:1.19.3-alpine AS builder -RUN apk add --no-cache git +RUN apk add --no-cache git gcc musl-dev RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest FROM alpine:3.16.3 From 45f9f5c199409aa5aee9cbe10626fb1c0485ec08 Mon Sep 17 00:00:00 2001 From: Sami <85764322+LuitelSamikshya@users.noreply.github.com> Date: Sun, 20 Nov 2022 05:32:24 -0600 Subject: [PATCH 317/355] Use a random hash instead of generating an url-based filename to store (#848) * Use a random hash instead of generating an url-based filename to store * switching to sha1 hash generation * removing max filename limit and reusing existing helper * hash for filename to store improvements * lint error fix * used filepath to be cross platform * used strings.Builder to prevent continous opening and closing of file * lint error fix * refactoring output logic * fixing lint errors * Using Sha1 to reduce file length Co-authored-by: mzack Co-authored-by: Mzack9999 Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- runner/options.go | 3 --- runner/runner.go | 48 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/runner/options.go b/runner/options.go index 81b5996..2136b08 100644 --- a/runner/options.go +++ b/runner/options.go @@ -28,8 +28,6 @@ import ( ) const ( - // The maximum file length is 251 (255 - 4 bytes for ".ext" suffix) - maxFileNameLength = 251 two = 2 DefaultResumeFile = "resume.cfg" DefaultOutputDirectory = "output" @@ -539,7 +537,6 @@ func (options *Options) ValidateOptions() error { gologger.Debug().Msgf("Store response directory specified, enabling \"sr\" flag automatically\n") options.StoreResponse = true } - if options.Hashes != "" { for _, hashType := range strings.Split(options.Hashes, ",") { if !slice.StringSliceContains([]string{"md5", "sha1", "sha256", "sha512", "mmh3", "simhash"}, strings.ToLower(hashType)) { diff --git a/runner/runner.go b/runner/runner.go index 7a429c2..1f7b3ff 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -91,7 +91,9 @@ func New(options *Options) (*Runner, error) { if err != nil { return nil, errors.Wrap(err, "could not create wappalyzer client") } - + if options.StoreResponseDir != "" { + os.RemoveAll(filepath.Join(options.StoreResponseDir, "index.txt")) + } dialerOpts := fastdialer.DefaultOptions dialerOpts.WithDialerHistory = true dialerOpts.MaxRetries = 3 @@ -588,7 +590,7 @@ func (r *Runner) RunEnumeration() { go func(output chan Result) { defer wgoutput.Done() - var f *os.File + var f, indexFile *os.File if r.options.Output != "" { var err error if r.options.Resume { @@ -597,7 +599,7 @@ func (r *Runner) RunEnumeration() { f, err = os.Create(r.options.Output) } if err != nil { - gologger.Fatal().Msgf("Could not create output file '%s': %s\n", r.options.Output, err) + gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", r.options.Output, err) } defer f.Close() //nolint } @@ -609,6 +611,19 @@ func (r *Runner) RunEnumeration() { f.WriteString(header + "\n") } } + if r.options.StoreResponseDir != "" { + var err error + indexPath := filepath.Join(r.options.StoreResponseDir, "index.txt") + if r.options.Resume { + indexFile, err = os.OpenFile(indexPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) + } else { + indexFile, err = os.Create(indexPath) + } + if err != nil { + gologger.Fatal().Msgf("Could not open/create index file '%s': %s\n", r.options.Output, err) + } + defer indexFile.Close() //nolint + } for resp := range output { if resp.err != nil { @@ -622,6 +637,11 @@ func (r *Runner) RunEnumeration() { continue } + if indexFile != nil { + indexData := fmt.Sprintf("%s %s (%d %s)\n", resp.StoredResponsePath, resp.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + _, _ = indexFile.WriteString(indexData) + } + // apply matchers and filters if r.options.OutputFilterCondition != "" || r.options.OutputMatchCondition != "" { rawMap, err := ResultToMap(resp) @@ -1518,28 +1538,26 @@ retry: var responsePath string if scanopts.StoreResponse || scanopts.StoreChain { domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".") - - // On various OS the file max file name length is 255 - https://serverfault.com/questions/9546/filename-length-limits-on-linux - // Truncating length at 255 - if len(domainFile) >= maxFileNameLength { - // leaving last 4 bytes free to append ".txt" - domainFile = domainFile[:maxFileNameLength] - } - - domainFile = strings.ReplaceAll(domainFile, "/", "[slash]") + ".txt" + hash := hashes.Sha1([]byte(domainFile)) + domainFile = fmt.Sprintf("%s.txt", hash) + domainBaseDir := filepath.Join(scanopts.StoreResponseDirectory, URL.Host) // store response - responsePath = filepath.Join(scanopts.StoreResponseDirectory, domainFile) + responsePath = filepath.Join(domainBaseDir, domainFile) respRaw := resp.Raw + reqRaw := requestDump if len(respRaw) > scanopts.MaxResponseBodySizeToSave { respRaw = respRaw[:scanopts.MaxResponseBodySizeToSave] } - writeErr := os.WriteFile(responsePath, []byte(respRaw), 0644) + data := append([]byte(fullURL), append([]byte("\n\n"), reqRaw...)...) + data = append(data, append([]byte("\n"), respRaw...)...) + _ = fileutil.CreateFolder(domainBaseDir) + writeErr := os.WriteFile(responsePath, data, 0644) if writeErr != nil { gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } if scanopts.StoreChain && resp.HasChain() { domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") - responsePath = filepath.Join(scanopts.StoreResponseDirectory, domainFile) + responsePath = filepath.Join(domainBaseDir, domainFile) writeErr := os.WriteFile(responsePath, []byte(resp.GetChain()), 0644) if writeErr != nil { gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) From a1f1fa897f41ae73db4f9c09d9df2cf4243dda82 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Sun, 20 Nov 2022 17:19:27 +0530 Subject: [PATCH 318/355] workflow update to avoid duplicate runs (#874) --- .github/workflows/build-test.yml | 1 - .github/workflows/codeql-analysis.yml | 1 - .github/workflows/functional-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release-binary.yml | 1 + .github/workflows/sonarcloud.yml | 6 +----- 6 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 74ccc30..54800ad 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,7 +1,6 @@ name: 🔨 Build Test on: - push: pull_request: workflow_dispatch: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2bbccfd..9f533f8 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,7 +2,6 @@ name: 🚨 CodeQL Analysis on: workflow_dispatch: - push: pull_request: branches: - dev diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 841226c..32e40cc 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -1,6 +1,6 @@ name: 🧪 Functional Test + on: - push: pull_request: workflow_dispatch: diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 40cef36..2e99e68 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -1,6 +1,6 @@ name: 🙏🏻 Lint Test + on: - push: pull_request: workflow_dispatch: diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 028de6e..5e99aaf 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -1,4 +1,5 @@ name: 🎉 Release Binary + on: create: tags: diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 20e7e48..3740833 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -1,11 +1,7 @@ name: 👮🏼‍♂️ Sonarcloud + on: - push: - branches: - - master - - dev pull_request: - types: [opened, synchronize, reopened] workflow_dispatch: jobs: From 0b08493e6db3c3970621fc5c4ad1adce075b8640 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Sun, 20 Nov 2022 17:32:38 +0530 Subject: [PATCH 319/355] broken link fix (#875) --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 461ddaf..a936317 100644 --- a/README.md +++ b/README.md @@ -520,6 +520,16 @@ func main() { # Acknowledgement -`httpx` is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/httpx/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow: +Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomnom/httprobe) work ❤️ -Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomnom/httprobe) work :heart: + +-------- + +
+ +`httpx` is made with 💙 by the [projectdiscovery](https://projectdiscovery.io) team and distributed under [MIT License](LICENSE.md). + + +Join Discord + +
From 294f1aa26676d7462501c8bc80cd16f7abd4ff4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 16:47:44 +0530 Subject: [PATCH 320/355] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.2 to 0.0.3 (#879) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.2 to 0.0.3. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.2...v0.0.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 27 +++-- go.sum | 378 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 364 insertions(+), 41 deletions(-) diff --git a/go.mod b/go.mod index f98b168..49b3960 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.3 github.com/projectdiscovery/gologger v1.1.4 - github.com/projectdiscovery/hmap v0.0.2 + github.com/projectdiscovery/hmap v0.0.3 github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 // indirect github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e @@ -59,33 +59,30 @@ require ( ) require ( - github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/DataDog/zstd v1.4.5 // indirect github.com/Knetic/govaluate v3.0.0+incompatible // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/cespare/xxhash v1.1.0 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/cockroachdb/errors v1.8.1 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect - github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae // indirect + github.com/cockroachdb/pebble v0.0.0-20221115214559-95662c240d17 // indirect github.com/cockroachdb/redact v1.0.8 // indirect github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/badger v1.6.2 // indirect - github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.11.7 // indirect + github.com/klauspost/compress v1.11.13 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -93,9 +90,21 @@ require ( github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.15 // indirect + github.com/prometheus/client_golang v1.12.0 // indirect + github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect + github.com/tidwall/btree v1.4.3 // indirect + github.com/tidwall/buntdb v1.2.10 // indirect + github.com/tidwall/gjson v1.14.3 // indirect + github.com/tidwall/grect v0.1.4 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/rtred v0.1.2 // indirect + github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect diff --git a/go.sum b/go.sum index f47128f..5598362 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,37 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= -github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= @@ -15,9 +44,6 @@ github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= -github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= @@ -26,6 +52,11 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -34,15 +65,23 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMfqcK8p0g= +github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bxcodec/faker/v4 v4.0.0-beta.3 h1:gqYNBvN72QtzKkYohNDKQlm+pg+uwBDVMN28nWHS18k= github.com/bxcodec/faker/v4 v4.0.0-beta.3/go.mod h1:m6+Ch1Lj3fqW/unZmvkXIdxWS5+XQWPWxcbbQW2X+Ho= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= @@ -53,13 +92,12 @@ github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQ github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae h1:rH4i31f3O0yEQy1Ho6DuUqf0+0Pq4MIflc8TBZEMvUI= -github.com/cockroachdb/pebble v0.0.0-20210728210723-48179f1d4dae/go.mod h1:JXfQr3d+XO4bL1pxGwKKo09xylQSdZ/mpZ9b2wfVcPs= +github.com/cockroachdb/pebble v0.0.0-20221115214559-95662c240d17 h1:D2i3P62ObgXAR0fC2j0qi2/hFnJv/MBpINAPiRqIN8I= +github.com/cockroachdb/pebble v0.0.0-20221115214559-95662c240d17/go.mod h1:qf9bLis2yy1XyNYD01wvIHPabuC1STzQsvGibYVsom4= github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -72,21 +110,14 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= -github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= -github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= @@ -97,36 +128,58 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -136,21 +189,43 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= @@ -158,22 +233,27 @@ github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= @@ -187,12 +267,13 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= -github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -211,6 +292,9 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= @@ -236,6 +320,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= @@ -246,18 +332,19 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -281,7 +368,6 @@ github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8E github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/fileutil v0.0.0-20220308101036-16c79af1cf5d/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= @@ -296,8 +382,8 @@ github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTt github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= -github.com/projectdiscovery/hmap v0.0.2 h1:fe3k0b6tj95mn9a1phD3JXvAAOmmWAh/Upg1Bf0Tfos= -github.com/projectdiscovery/hmap v0.0.2/go.mod h1:YU3TeNTDmLW2dtb4LvuEtDsPTsQ06XMnmOeD3KOuU6c= +github.com/projectdiscovery/hmap v0.0.3 h1:7v1cltUmfbOySZBopSOWlkGyVyW9ftxofY09Bz+lctk= +github.com/projectdiscovery/hmap v0.0.3/go.mod h1:uYaZSMeXPu5QjDInvtGd/KxTnvF709a0LroIb+FEh1E= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 h1:HRqev12wKvcwK1fe4pSlMfQdPHo9LfTxuFeRN4f3tS4= @@ -326,7 +412,6 @@ github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI= github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= @@ -338,9 +423,32 @@ github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0d github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= github.com/projectdiscovery/wappalyzergo v0.0.69 h1:x5etq0b+bxQg3dhbSfOEYoV5ODl6M+S99KxyXLdkuFY= github.com/projectdiscovery/wappalyzergo v0.0.69/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg= +github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= @@ -353,14 +461,16 @@ github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -377,7 +487,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= @@ -387,6 +496,25 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= +github.com/tidwall/btree v1.4.3 h1:Lf5U/66bk0ftNppOBjVoy/AIPBrLMkheBp4NnSNiYOo= +github.com/tidwall/btree v1.4.3/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= +github.com/tidwall/buntdb v1.2.10 h1:U/ebfkmYPBnyiNZIirUiWFcxA/mgzjbKlyPynFsPtyM= +github.com/tidwall/buntdb v1.2.10/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= +github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= +github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= +github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= +github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= +github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= +github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= +github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -411,7 +539,9 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/ github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -425,6 +555,11 @@ github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWV go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= @@ -434,6 +569,8 @@ go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -444,15 +581,32 @@ golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -464,19 +618,38 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -484,6 +657,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -495,25 +669,40 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -521,20 +710,38 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -546,26 +753,64 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -577,26 +822,86 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -614,6 +919,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -622,4 +928,12 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 8a3d65886d225d9b3dee03b554bbf73053240f29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 11:18:39 +0000 Subject: [PATCH 321/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.69 to 0.0.70. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.69...v0.0.70) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 49b3960..7c8b2bc 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/projectdiscovery/wappalyzergo v0.0.69 + github.com/projectdiscovery/wappalyzergo v0.0.70 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 5598362..abde5f1 100644 --- a/go.sum +++ b/go.sum @@ -421,8 +421,8 @@ github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjb github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0dudpcGfM= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= -github.com/projectdiscovery/wappalyzergo v0.0.69 h1:x5etq0b+bxQg3dhbSfOEYoV5ODl6M+S99KxyXLdkuFY= -github.com/projectdiscovery/wappalyzergo v0.0.69/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/wappalyzergo v0.0.70 h1:0Q4KmgHGKqyfbSrj3EUGudv+tXrLnS5/jX92r0BnQtE= +github.com/projectdiscovery/wappalyzergo v0.0.70/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 4ddabc76750eda200b5c2c43064d20f1994283ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 11:18:39 +0000 Subject: [PATCH 322/355] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.17 to 0.0.18. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.17...v0.0.18) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 9 ++++----- go.sum | 58 +++++++++------------------------------------------------- 2 files changed, 13 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 49b3960..5de0c1a 100644 --- a/go.mod +++ b/go.mod @@ -15,19 +15,18 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.9 - github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 // indirect - github.com/projectdiscovery/fastdialer v0.0.17 + github.com/projectdiscovery/fastdialer v0.0.18 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.3 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.3 - github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 // indirect + github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d - github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 // indirect + github.com/projectdiscovery/sliceutil v0.0.1 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect github.com/projectdiscovery/wappalyzergo v0.0.69 github.com/remeh/sizedwaitgroup v1.0.0 @@ -89,7 +88,7 @@ require ( github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect - github.com/projectdiscovery/retryabledns v1.0.15 // indirect + github.com/projectdiscovery/retryabledns v1.0.17 // indirect github.com/prometheus/client_golang v1.12.0 // indirect github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect github.com/prometheus/common v0.32.1 // indirect diff --git a/go.sum b/go.sum index 5598362..c5d94c5 100644 --- a/go.sum +++ b/go.sum @@ -49,7 +49,6 @@ github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejn github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -185,7 +184,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= @@ -257,7 +255,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= @@ -300,8 +297,6 @@ github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQ github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= -github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -325,19 +320,16 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -352,48 +344,38 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/asnmap v0.0.1 h1:n4YCz1ljUaDA3dOUCkjI/bUOtiS7ge1KJ39qpURCd/o= github.com/projectdiscovery/asnmap v0.0.1/go.mod h1:CjCVDhQPVtmlE247L6YFeIVX9c4m8pOX8V8BmB0JkX8= -github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK68yPOnNNRmwuXqytl+T9EbwneEUCvMDRjLe0J04= github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.9 h1:8sA17+2qP+cTnY7LaaGURJW5stSM8UwQiygwHQjxHx4= github.com/projectdiscovery/clistats v0.0.9/go.mod h1:7F1RdeGAoLf05rhsZesL0+qoXJpOA/vxuRj2JRIAzU4= -github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= -github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= github.com/projectdiscovery/dsl v0.0.3 h1:oWlZZaSADqoyfJdUHWqAzpB65NpvLukZQGFv1uTtU3g= github.com/projectdiscovery/dsl v0.0.3/go.mod h1:ST66slxtp7fsFqTOq3k6+1TDV7RH1Moz3sPBqUGn4Fg= -github.com/projectdiscovery/fastdialer v0.0.17 h1:wT7LinZAcyEzkfhJLhWNQrA9m79+hTd6CJ7aWTJfXjk= -github.com/projectdiscovery/fastdialer v0.0.17/go.mod h1:poZbCGYGRfRcKxU8/1rWH8EVOJ4AZn1a8+CyINHzffQ= +github.com/projectdiscovery/fastdialer v0.0.18 h1:S7PjMwtyaoHMNcKGjnRgTLChp+07Juay3IzbEnd0va0= +github.com/projectdiscovery/fastdialer v0.0.18/go.mod h1:KWGsYRSLSlF/jMQW/P2Dar3gJhrkiDKF0zkKtXzcKmI= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= -github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= -github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M= github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= -github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa/go.mod h1:lV5f/PNPmCCjCN/dR317/chN9s7VG5h/xcbFfXOz8Fo= github.com/projectdiscovery/hmap v0.0.3 h1:7v1cltUmfbOySZBopSOWlkGyVyW9ftxofY09Bz+lctk= github.com/projectdiscovery/hmap v0.0.3/go.mod h1:uYaZSMeXPu5QjDInvtGd/KxTnvF709a0LroIb+FEh1E= github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= -github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 h1:HRqev12wKvcwK1fe4pSlMfQdPHo9LfTxuFeRN4f3tS4= -github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1:vHRC+9exsfSbEngMKDl0xiWqkxlLk3lHQZpbS2yFT8U= +github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= +github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= -github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= -github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/ratelimit v0.0.1 h1:GnCfbKmkLdDLXT3QS4KS0zCsuDGkoRQE0YDbTqzQmS8= @@ -402,19 +384,13 @@ github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpG github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f/go.mod h1:3L0WfNIcVWXIDur8k+gKDLZLWY2F+rs0SQXtcn/3AYU= -github.com/projectdiscovery/retryabledns v1.0.15 h1:3Nn119UwYsfUPC3g0q57ftz0Wb5Zl5ppvw8R0Xu0DEI= -github.com/projectdiscovery/retryabledns v1.0.15/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk= +github.com/projectdiscovery/retryabledns v1.0.17 h1:XKzI26UKYt2g7YLJ/EcyYmM04sfD1vurETecPEpeA1w= +github.com/projectdiscovery/retryabledns v1.0.17/go.mod h1:Dyhq/f0sGmXueso0+Ah3LbJfsX4PXpBrpfiyjZZ8SDk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= -github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= -github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA= -github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= -github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI= -github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= -github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= +github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyBSNkVahatleg= +github.com/projectdiscovery/sliceutil v0.0.1/go.mod h1:0wBmhU5uTDwMfrEZfvwH9qa5k60Q4shPVOC9E6LGsDI= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= @@ -489,8 +465,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -525,7 +499,6 @@ github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBn github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.15.1-0.20210928183822-5ee35905bd95/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 h1:oRCu5zb6sklsDvy5sOz3dFqGg5vAEYBBD2MAYhNThCQ= github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= @@ -549,10 +522,8 @@ github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq9 github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 h1:17HHAgFKlLcZsDOjBOUrd5hDihb1ggf+1a5dTbkgkIY= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b/go.mod h1:5nID//bFGkx3/+iHcFIFRHQ54EOPJ0iSj0IGKpMElvw= github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be h1:M5QjuCbUeNZsup53dlJkI/cx6pVdnDOPzyy+XppoowY= github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -561,7 +532,6 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= @@ -652,18 +622,13 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= @@ -732,10 +697,8 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -745,7 +708,6 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -811,7 +773,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -821,7 +782,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= From f82f48b3c42a0a00c36a513a55eba4c58434161f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Nov 2022 08:17:01 +0000 Subject: [PATCH 323/355] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.3 to 0.1.5. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.3...v0.1.5) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c875109..04f8591 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.1.3 + github.com/projectdiscovery/goflags v0.1.5 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.3 github.com/projectdiscovery/iputil v0.0.2 // indirect diff --git a/go.sum b/go.sum index 898efb7..454f769 100644 --- a/go.sum +++ b/go.sum @@ -360,8 +360,8 @@ github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJw github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M= -github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q= +github.com/projectdiscovery/goflags v0.1.5 h1:zqZvIKAdSC62e5jv5UDgBWyP61KrLHxxUXjH/MAJlB4= +github.com/projectdiscovery/goflags v0.1.5/go.mod h1:mms5rLXW0+jjbFFTyl/4t9VwtP5TCE+1Y1pReezluAI= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= From 68fafc24c1739046b8ce3ca8bbf2ac33ce5b2d01 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Nov 2022 08:31:31 +0000 Subject: [PATCH 324/355] chore(deps): bump github.com/projectdiscovery/gologger Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.4 to 1.1.5. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.4...v1.1.5) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 9 ++++++++- go.sum | 24 +++++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 04f8591..e7ab354 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.5 - github.com/projectdiscovery/gologger v1.1.4 + github.com/projectdiscovery/gologger v1.1.5 github.com/projectdiscovery/hmap v0.0.3 github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/mapcidr v1.0.3 @@ -74,6 +74,7 @@ require ( github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/dsnet/compress v0.0.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect @@ -82,8 +83,11 @@ require ( github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mholt/archiver v3.1.1+incompatible // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect + github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect @@ -104,8 +108,10 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect + github.com/ulikunitz/xz v0.5.7 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect @@ -114,6 +120,7 @@ require ( golang.org/x/mod v0.6.0 // indirect golang.org/x/tools v0.2.0 // indirect google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 454f769..7f62a6d 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,9 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -123,6 +126,8 @@ github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHj github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= @@ -183,6 +188,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -262,10 +268,12 @@ github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiD github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -296,6 +304,8 @@ github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go. github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= +github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= +github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= @@ -320,6 +330,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -333,6 +345,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= +github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -363,8 +377,9 @@ github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h github.com/projectdiscovery/goflags v0.1.5 h1:zqZvIKAdSC62e5jv5UDgBWyP61KrLHxxUXjH/MAJlB4= github.com/projectdiscovery/goflags v0.1.5/go.mod h1:mms5rLXW0+jjbFFTyl/4t9VwtP5TCE+1Y1pReezluAI= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= -github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= +github.com/projectdiscovery/gologger v1.1.5 h1:CT2Jrpzusca+dwNREMMz4RnAreJ0RqaISRU4SBcZC+M= +github.com/projectdiscovery/gologger v1.1.5/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/hmap v0.0.3 h1:7v1cltUmfbOySZBopSOWlkGyVyW9ftxofY09Bz+lctk= github.com/projectdiscovery/hmap v0.0.3/go.mod h1:uYaZSMeXPu5QjDInvtGd/KxTnvF709a0LroIb+FEh1E= @@ -491,6 +506,9 @@ github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaym github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= @@ -505,6 +523,8 @@ github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= @@ -867,6 +887,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= +gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= From 20a9a930c20c210e2f0d41cd5489789fb83a316f Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Thu, 24 Nov 2022 21:50:37 +0530 Subject: [PATCH 325/355] adds base64 encoding support (#813) (#885) * adds base64 encoding support (#813) * rename b64 to base64 --- common/stringz/stringz.go | 5 +++++ runner/options.go | 4 ++++ runner/runner.go | 6 ++++++ 3 files changed, 15 insertions(+) diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index 8d8a140..d463804 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -127,3 +127,8 @@ func InsertInto(s string, interval int, sep rune) string { buffer.WriteRune(sep) return buffer.String() } + +// Base64 returns base64 of given byte array +func Base64(bin []byte) string { + return base64.StdEncoding.EncodeToString(bin) +} diff --git a/runner/options.go b/runner/options.go index 2136b08..665bd84 100644 --- a/runner/options.go +++ b/runner/options.go @@ -54,6 +54,7 @@ type scanOptions struct { OutputWithNoColor bool OutputMethod bool ResponseInStdout bool + Base64ResponseInStdout bool ChainInStdout bool TLSProbe bool CSPProbe bool @@ -102,6 +103,7 @@ func (s *scanOptions) Clone() *scanOptions { OutputWithNoColor: s.OutputWithNoColor, OutputMethod: s.OutputMethod, ResponseInStdout: s.ResponseInStdout, + Base64ResponseInStdout: s.Base64ResponseInStdout, ChainInStdout: s.ChainInStdout, TLSProbe: s.TLSProbe, CSPProbe: s.CSPProbe, @@ -184,6 +186,7 @@ type Options struct { OutputServerHeader bool OutputWebSocket bool responseInStdout bool + base64responseInStdout bool chainInStdout bool FollowHostRedirects bool MaxRedirects int @@ -350,6 +353,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"), flagSet.BoolVar(&options.JSONOutput, "json", false, "store output in JSONL(ines) format"), flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"), + flagSet.BoolVarP(&options.base64responseInStdout, "include-response-base64", "irrb", false, "include base64 encoded http request/response in JSON output (-json only)"), flagSet.BoolVar(&options.chainInStdout, "include-chain", false, "include redirect http chain in JSON output (-json only)"), flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"), ) diff --git a/runner/runner.go b/runner/runner.go index 1f7b3ff..0e434e3 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -216,6 +216,7 @@ func New(options *Options) (*Runner, error) { scanopts.OutputServerHeader = options.OutputServerHeader scanopts.OutputWithNoColor = options.NoColor scanopts.ResponseInStdout = options.responseInStdout + scanopts.Base64ResponseInStdout = options.base64responseInStdout scanopts.ChainInStdout = options.chainInStdout scanopts.OutputWebSocket = options.OutputWebSocket scanopts.TLSProbe = options.TLSProbe @@ -1292,6 +1293,11 @@ retry: request = string(requestDump) responseHeader = normalizeHeaders(resp.Headers) rawResponseHeader = resp.RawHeaders + } else if scanopts.Base64ResponseInStdout { + serverResponseRaw = stringz.Base64(resp.Data) + request = stringz.Base64(requestDump) + responseHeader = normalizeHeaders(resp.Headers) + rawResponseHeader = stringz.Base64([]byte(resp.RawHeaders)) } // check for virtual host From 8c40e52322bf390232900d6955d349723d6d89bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 17:16:03 +0530 Subject: [PATCH 326/355] chore(deps): bump alpine from 3.16.3 to 3.17.0 (#888) Bumps alpine from 3.16.3 to 3.17.0. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f2150e7..18ec5a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM golang:1.19.3-alpine AS builder RUN apk add --no-cache git gcc musl-dev RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.16.3 +FROM alpine:3.17.0 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ From 8bd903448a7f349c0a9b1b5e839b3ae032d911e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 17:22:12 +0530 Subject: [PATCH 327/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.70 to 0.0.71 (#889) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.70 to 0.0.71. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.70...v0.0.71) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e7ab354..5de7b7d 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.1 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/projectdiscovery/wappalyzergo v0.0.70 + github.com/projectdiscovery/wappalyzergo v0.0.71 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 7f62a6d..5ca90b6 100644 --- a/go.sum +++ b/go.sum @@ -412,8 +412,8 @@ github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjb github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0dudpcGfM= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= -github.com/projectdiscovery/wappalyzergo v0.0.70 h1:0Q4KmgHGKqyfbSrj3EUGudv+tXrLnS5/jX92r0BnQtE= -github.com/projectdiscovery/wappalyzergo v0.0.70/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/wappalyzergo v0.0.71 h1:NEtPLEUx21rhE2H89Pc9RxImrDIG90CNXLRoSDwtiSA= +github.com/projectdiscovery/wappalyzergo v0.0.71/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From b8de450be6a0186a8f74bf73e0797f81da99f405 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Tue, 29 Nov 2022 07:48:38 +0100 Subject: [PATCH 328/355] bom utf8 on csv output (#887) * implemented bom utf8 on csv * added on csvoutput * Removed debug code * added output encoding flag * using strings utils * small refactor * removing default encoding Co-authored-by: mzack --- runner/options.go | 5 +++++ runner/runner.go | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/runner/options.go b/runner/options.go index 665bd84..c3db6fd 100644 --- a/runner/options.go +++ b/runner/options.go @@ -179,6 +179,7 @@ type Options struct { StoreResponse bool JSONOutput bool CSVOutput bool + CSVOutputEncoding string Silent bool Version bool Verbose bool @@ -351,6 +352,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http response to output directory"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http response to custom directory"), flagSet.BoolVar(&options.CSVOutput, "csv", false, "store output in csv format"), + flagSet.StringVarP(&options.CSVOutputEncoding, "csv-output-encoding", "csvo", "", "define output encoding"), flagSet.BoolVar(&options.JSONOutput, "json", false, "store output in JSONL(ines) format"), flagSet.BoolVarP(&options.responseInStdout, "include-response", "irr", false, "include http request/response in JSON output (-json only)"), flagSet.BoolVarP(&options.base64responseInStdout, "include-response-base64", "irrb", false, "include base64 encoded http request/response in JSON output (-json only)"), @@ -573,6 +575,9 @@ func (options *Options) configureOutput() { if len(options.OutputMatchResponseTime) > 0 || len(options.OutputFilterResponseTime) > 0 { options.OutputResponseTime = true } + if options.CSVOutputEncoding != "" { + options.CSVOutput = true + } } func (options *Options) configureResume() error { diff --git a/runner/runner.go b/runner/runner.go index 0e434e3..1d3eb19 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -592,6 +592,7 @@ func (r *Runner) RunEnumeration() { defer wgoutput.Done() var f, indexFile *os.File + if r.options.Output != "" { var err error if r.options.Resume { @@ -605,6 +606,18 @@ func (r *Runner) RunEnumeration() { defer f.Close() //nolint } if r.options.CSVOutput { + outEncoding := strings.ToLower(r.options.CSVOutputEncoding) + switch outEncoding { + case "": // no encoding do nothing + case "utf-8", "utf8": + bomUtf8 := []byte{0xEF, 0xBB, 0xBF} + _, err := f.Write(bomUtf8) + if err != nil { + gologger.Fatal().Msgf("err on file write: %s\n", err) + } + default: // unknown encoding + gologger.Fatal().Msgf("unknown csv output encoding: %s\n", r.options.CSVOutputEncoding) + } header := Result{}.CSVHeader() gologger.Silent().Msgf("%s\n", header) if f != nil { @@ -797,6 +810,7 @@ func (r *Runner) RunEnumeration() { row = resp.JSON(&r.scanopts) } else if r.options.CSVOutput { row = resp.CSVRow(&r.scanopts) + } gologger.Silent().Msgf("%s\n", row) From e64587292ec60a7e77583038ea5c762465cd827b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 12:19:22 +0530 Subject: [PATCH 329/355] chore(deps): bump github.com/projectdiscovery/ratelimit from 0.0.1 to 0.0.2 (#890) * chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.1 to 0.0.2. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.1...v0.0.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * fixing type Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mzack --- go.mod | 2 +- go.sum | 4 ++-- runner/runner.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 5de7b7d..f8691c3 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/asnmap v0.0.1 github.com/projectdiscovery/dsl v0.0.3 - github.com/projectdiscovery/ratelimit v0.0.1 + github.com/projectdiscovery/ratelimit v0.0.2 github.com/projectdiscovery/tlsx v0.0.9 github.com/projectdiscovery/utils v0.0.3 github.com/stretchr/testify v1.8.1 diff --git a/go.sum b/go.sum index 5ca90b6..0c1a892 100644 --- a/go.sum +++ b/go.sum @@ -393,8 +393,8 @@ github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgy github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= -github.com/projectdiscovery/ratelimit v0.0.1 h1:GnCfbKmkLdDLXT3QS4KS0zCsuDGkoRQE0YDbTqzQmS8= -github.com/projectdiscovery/ratelimit v0.0.1/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= +github.com/projectdiscovery/ratelimit v0.0.2 h1:vRscyCcmTWndd16Ed3lC51196eKYena+xpYfrDomjMg= +github.com/projectdiscovery/ratelimit v0.0.2/go.mod h1:WBz8N1P+CyxnfUoGfVCqah4NZ2SreSX7v9dY8wIlK70= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= diff --git a/runner/runner.go b/runner/runner.go index 1d3eb19..cb9a07f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -298,9 +298,9 @@ func New(options *Options) (*Runner, error) { runner.hm = hm if options.RateLimitMinute > 0 { - runner.ratelimiter = *ratelimit.New(context.Background(), int64(options.RateLimitMinute), time.Minute) + runner.ratelimiter = *ratelimit.New(context.Background(), uint(options.RateLimitMinute), time.Minute) } else if options.RateLimit > 0 { - runner.ratelimiter = *ratelimit.New(context.Background(), int64(options.RateLimit), time.Second) + runner.ratelimiter = *ratelimit.New(context.Background(), uint(options.RateLimit), time.Second) } else { runner.ratelimiter = *ratelimit.NewUnlimited(context.Background()) } From a9c35951fa5fd54c9ba7342236ab315e8d432014 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Dec 2022 10:04:57 +0000 Subject: [PATCH 330/355] chore(deps): bump github.com/projectdiscovery/fastdialer Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.18 to 0.0.19. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.18...v0.0.19) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 18 ++++-------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index f8691c3..b7f88cd 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.9 - github.com/projectdiscovery/fastdialer v0.0.18 + github.com/projectdiscovery/fastdialer v0.0.19 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c @@ -90,7 +90,7 @@ require ( github.com/pierrec/lz4 v2.6.0+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect - github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d // indirect + github.com/projectdiscovery/networkpolicy v0.0.3 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.17 // indirect github.com/prometheus/client_golang v1.12.0 // indirect diff --git a/go.sum b/go.sum index 0c1a892..24c84e8 100644 --- a/go.sum +++ b/go.sum @@ -189,7 +189,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= @@ -288,7 +287,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -366,8 +364,8 @@ github.com/projectdiscovery/clistats v0.0.9 h1:8sA17+2qP+cTnY7LaaGURJW5stSM8UwQi github.com/projectdiscovery/clistats v0.0.9/go.mod h1:7F1RdeGAoLf05rhsZesL0+qoXJpOA/vxuRj2JRIAzU4= github.com/projectdiscovery/dsl v0.0.3 h1:oWlZZaSADqoyfJdUHWqAzpB65NpvLukZQGFv1uTtU3g= github.com/projectdiscovery/dsl v0.0.3/go.mod h1:ST66slxtp7fsFqTOq3k6+1TDV7RH1Moz3sPBqUGn4Fg= -github.com/projectdiscovery/fastdialer v0.0.18 h1:S7PjMwtyaoHMNcKGjnRgTLChp+07Juay3IzbEnd0va0= -github.com/projectdiscovery/fastdialer v0.0.18/go.mod h1:KWGsYRSLSlF/jMQW/P2Dar3gJhrkiDKF0zkKtXzcKmI= +github.com/projectdiscovery/fastdialer v0.0.19 h1:0E9trACMtYq1JgkY+sM8b6XABjITRwHBSWwhH9csmgY= +github.com/projectdiscovery/fastdialer v0.0.19/go.mod h1:9zV1eivctLQkBUykwBj5BaPkYz8A8j1Sou2UCgdCQ5I= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= @@ -376,23 +374,16 @@ github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.1.5 h1:zqZvIKAdSC62e5jv5UDgBWyP61KrLHxxUXjH/MAJlB4= github.com/projectdiscovery/goflags v0.1.5/go.mod h1:mms5rLXW0+jjbFFTyl/4t9VwtP5TCE+1Y1pReezluAI= -github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= -github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= github.com/projectdiscovery/gologger v1.1.5 h1:CT2Jrpzusca+dwNREMMz4RnAreJ0RqaISRU4SBcZC+M= github.com/projectdiscovery/gologger v1.1.5/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= -github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/hmap v0.0.3 h1:7v1cltUmfbOySZBopSOWlkGyVyW9ftxofY09Bz+lctk= github.com/projectdiscovery/hmap v0.0.3/go.mod h1:uYaZSMeXPu5QjDInvtGd/KxTnvF709a0LroIb+FEh1E= -github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= -github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= -github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= -github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= -github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d h1:QXaK3yzoEWI8n+hLAqEgTJEWhkp1WZM8ThbKwrlXFks= -github.com/projectdiscovery/networkpolicy v0.0.2-0.20220525172507-b844eafc878d/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= +github.com/projectdiscovery/networkpolicy v0.0.3 h1:OZFPkMVY6SJxc1ncuRXB2VlT6xlzJl5eWDkvpWn/GMM= +github.com/projectdiscovery/networkpolicy v0.0.3/go.mod h1:DIXwKs3sQyfCoWHKRLQiRrEorSQW4Zrh4ftu7oDVK6w= github.com/projectdiscovery/ratelimit v0.0.2 h1:vRscyCcmTWndd16Ed3lC51196eKYena+xpYfrDomjMg= github.com/projectdiscovery/ratelimit v0.0.2/go.mod h1:WBz8N1P+CyxnfUoGfVCqah4NZ2SreSX7v9dY8wIlK70= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= @@ -477,7 +468,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= From a4097e4c1a9135d91d76db0f35f05c051851a947 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Dec 2022 10:05:03 +0000 Subject: [PATCH 331/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.71 to 0.0.72. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.71...v0.0.72) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f8691c3..788f933 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.1 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/projectdiscovery/wappalyzergo v0.0.71 + github.com/projectdiscovery/wappalyzergo v0.0.72 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 0c1a892..f0031ca 100644 --- a/go.sum +++ b/go.sum @@ -412,8 +412,8 @@ github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjb github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0dudpcGfM= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= -github.com/projectdiscovery/wappalyzergo v0.0.71 h1:NEtPLEUx21rhE2H89Pc9RxImrDIG90CNXLRoSDwtiSA= -github.com/projectdiscovery/wappalyzergo v0.0.71/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/wappalyzergo v0.0.72 h1:Lb8+Ij9zfFHXBJJR8hZDG8TkwnMXvcu5/vWxO1rVKn0= +github.com/projectdiscovery/wappalyzergo v0.0.72/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From f08e5d3e281c05230c5d9d4c7f3cc76f7c1d821a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Dec 2022 10:03:07 +0000 Subject: [PATCH 332/355] chore(deps): bump golang.org/x/net from 0.2.0 to 0.4.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.2.0 to 0.4.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.2.0...v0.4.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f8691c3..66b6f3b 100644 --- a/go.mod +++ b/go.mod @@ -34,9 +34,9 @@ require ( github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/net v0.2.0 - golang.org/x/sys v0.2.0 // indirect - golang.org/x/text v0.4.0 + golang.org/x/net v0.4.0 + golang.org/x/sys v0.3.0 // indirect + golang.org/x/text v0.5.0 ) require github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index 0c1a892..9329114 100644 --- a/go.sum +++ b/go.sum @@ -651,8 +651,8 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -729,8 +729,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -742,8 +742,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From a3dd154180519c5c71765c7254f68526f773bc9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 16:39:35 +0530 Subject: [PATCH 333/355] chore(deps): bump github.com/projectdiscovery/tlsx from 0.0.9 to 1.0.0 (#898) Bumps [github.com/projectdiscovery/tlsx](https://github.com/projectdiscovery/tlsx) from 0.0.9 to 1.0.0. - [Release notes](https://github.com/projectdiscovery/tlsx/releases) - [Changelog](https://github.com/projectdiscovery/tlsx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/tlsx/compare/v0.0.9...v1.0.0) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/tlsx dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cc10855..ac501e6 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/projectdiscovery/asnmap v0.0.1 github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/ratelimit v0.0.2 - github.com/projectdiscovery/tlsx v0.0.9 + github.com/projectdiscovery/tlsx v1.0.0 github.com/projectdiscovery/utils v0.0.3 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.8.0 diff --git a/go.sum b/go.sum index 65b2efd..14f0e41 100644 --- a/go.sum +++ b/go.sum @@ -399,8 +399,8 @@ github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyB github.com/projectdiscovery/sliceutil v0.0.1/go.mod h1:0wBmhU5uTDwMfrEZfvwH9qa5k60Q4shPVOC9E6LGsDI= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v0.0.9 h1:wUC8GYUIo5jd+enqE1lnEJ3Ew7m+N6eRmFBjbSJLomU= -github.com/projectdiscovery/tlsx v0.0.9/go.mod h1:bPKwgeGRMZaDpOQCy6TjQWr3bQ7d9lW2lVH5BnWlWMI= +github.com/projectdiscovery/tlsx v1.0.0 h1:krS0QRbh4wHqeOybVQwBImY9PmIqzFOYM9Q+cKlEiZE= +github.com/projectdiscovery/tlsx v1.0.0/go.mod h1:LOgQpCTE96d/18+Zwu8p5b8ujOGPn6lIDqMIQCJHoXM= github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0dudpcGfM= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= github.com/projectdiscovery/wappalyzergo v0.0.72 h1:Lb8+Ij9zfFHXBJJR8hZDG8TkwnMXvcu5/vWxO1rVKn0= From 367ceb927275655280057a3bab20655464f7ee2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 10:03:32 +0000 Subject: [PATCH 334/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.72 to 0.0.73. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.72...v0.0.73) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ac501e6..1c3133e 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.1 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/projectdiscovery/wappalyzergo v0.0.72 + github.com/projectdiscovery/wappalyzergo v0.0.73 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index 14f0e41..29c60bf 100644 --- a/go.sum +++ b/go.sum @@ -403,8 +403,8 @@ github.com/projectdiscovery/tlsx v1.0.0 h1:krS0QRbh4wHqeOybVQwBImY9PmIqzFOYM9Q+c github.com/projectdiscovery/tlsx v1.0.0/go.mod h1:LOgQpCTE96d/18+Zwu8p5b8ujOGPn6lIDqMIQCJHoXM= github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0dudpcGfM= github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= -github.com/projectdiscovery/wappalyzergo v0.0.72 h1:Lb8+Ij9zfFHXBJJR8hZDG8TkwnMXvcu5/vWxO1rVKn0= -github.com/projectdiscovery/wappalyzergo v0.0.72/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/wappalyzergo v0.0.73 h1:+ctoMja3kmALX1MDcpDsoqY3cU83hd3/fBSi7dm8y5M= +github.com/projectdiscovery/wappalyzergo v0.0.73/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From cabd55a0a8ca3feb5f69d055cee7d6dde3276ca6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Dec 2022 22:31:34 +0530 Subject: [PATCH 335/355] chore(deps): bump golang from 1.19.3-alpine to 1.19.4-alpine (#900) Bumps golang from 1.19.3-alpine to 1.19.4-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 18ec5a2..ddfb4b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.3-alpine AS builder +FROM golang:1.19.4-alpine AS builder RUN apk add --no-cache git gcc musl-dev RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest From d2476b282dd8c7638e6e31ee78898ea39e9a3149 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Dec 2022 10:03:16 +0000 Subject: [PATCH 336/355] chore(deps): bump go.uber.org/multierr from 1.8.0 to 1.9.0 Bumps [go.uber.org/multierr](https://github.com/uber-go/multierr) from 1.8.0 to 1.9.0. - [Release notes](https://github.com/uber-go/multierr/releases) - [Changelog](https://github.com/uber-go/multierr/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/multierr/compare/v1.8.0...v1.9.0) --- updated-dependencies: - dependency-name: go.uber.org/multierr dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ac501e6..5ba0970 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/projectdiscovery/tlsx v1.0.0 github.com/projectdiscovery/utils v0.0.3 github.com/stretchr/testify v1.8.1 - go.uber.org/multierr v1.8.0 + go.uber.org/multierr v1.9.0 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 ) diff --git a/go.sum b/go.sum index 14f0e41..4f4e034 100644 --- a/go.sum +++ b/go.sum @@ -541,11 +541,10 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= From fb24f546be7a05564613959f269621cde8c85031 Mon Sep 17 00:00:00 2001 From: Nicholas de Jong Date: Sat, 17 Dec 2022 22:02:44 +1000 Subject: [PATCH 337/355] =?UTF-8?q?Provides=20BUILD=5FSOURCE=5FTAG=20as=20?= =?UTF-8?q?an=20ARG=20in=20the=20Dockerfile=20that=20hence=20allo=E2=80=A6?= =?UTF-8?q?=20(#905)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provides BUILD_SOURCE_TAG as an ARG in the Dockerfile that hence allows Docker container builders to target specific tagged-versions other than latest Co-authored-by: Sandeep Singh Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ddfb4b0..1ac3bf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM golang:1.19.4-alpine AS builder -RUN apk add --no-cache git gcc musl-dev -RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest +ARG BUILD_SOURCE_TAG=latest +RUN apk add --no-cache git build-base gcc musl-dev +RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@${BUILD_SOURCE_TAG} FROM alpine:3.17.0 RUN apk -U upgrade --no-cache \ From 986b2184c567e165ffe67c00e30a043a57092de3 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 23 Dec 2022 18:56:38 +0100 Subject: [PATCH 338/355] Enforcing wildcard check to prefix only (#911) enforcing wildcard check to prefix only --- cmd/functional-test/testcases.txt | 3 ++- runner/runner.go | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index 476c901..0121c7c 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -14,4 +14,5 @@ scanme.sh {{binary}} -silent -unsafe scanme.sh {{binary}} -silent -x all scanme.sh {{binary}} -silent -body 'a=b' scanme.sh {{binary}} -silent -exclude-cdn -scanme.sh {{binary}} -silent -ports https:443 \ No newline at end of file +scanme.sh {{binary}} -silent -ports https:443 +https://scanme.sh?a=1*1 {{binary}} -silent \ No newline at end of file diff --git a/runner/runner.go b/runner/runner.go index cb9a07f..6258ed7 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -964,13 +964,14 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results := make(chan httpx.Target) go func() { defer close(results) + + target = strings.TrimSpace(target) + switch { - case strings.ContainsAny(target, "*") || strings.HasPrefix(target, "."): + case stringsutil.HasPrefixAny(target, "*", "."): // A valid target does not contain: - // * - // spaces // trim * and/or . (prefix) from the target to return the domain instead of wilcard - target = strings.TrimPrefix(strings.Trim(target, "*"), ".") + target = stringsutil.TrimPrefixAny(target, "*", ".") if !r.testAndSet(target) { return } From bcebef30b31ab098b1fa39ae170ee92323b62f76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Dec 2022 03:05:53 +0530 Subject: [PATCH 339/355] chore(deps): bump goreleaser/goreleaser-action from 3 to 4 (#907) Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 3 to 4. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/v3...v4) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-binary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 5e99aaf..98365c7 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -21,7 +21,7 @@ jobs: go-version: 1.18 - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v3 + uses: goreleaser/goreleaser-action@v4 with: args: "release --rm-dist" version: latest From c483af1bfab7e355a55dc3d5b218e412a0f3232b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Dec 2022 03:06:05 +0530 Subject: [PATCH 340/355] chore(deps): bump github.com/projectdiscovery/goflags from 0.1.5 to 0.1.6 (#910) chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.5 to 0.1.6. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.5...v0.1.6) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index a2d85b2..e5dd954 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.1.5 + github.com/projectdiscovery/goflags v0.1.6 github.com/projectdiscovery/gologger v1.1.5 github.com/projectdiscovery/hmap v0.0.3 github.com/projectdiscovery/iputil v0.0.2 // indirect @@ -51,7 +51,7 @@ require ( github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/ratelimit v0.0.2 github.com/projectdiscovery/tlsx v1.0.0 - github.com/projectdiscovery/utils v0.0.3 + github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.9.0 golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 @@ -80,7 +80,7 @@ require ( github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.11.13 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect @@ -97,7 +97,7 @@ require ( github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/rogpeppe/go-internal v1.8.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tidwall/btree v1.4.3 // indirect diff --git a/go.sum b/go.sum index 70f460d..884f8af 100644 --- a/go.sum +++ b/go.sum @@ -279,8 +279,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -372,8 +372,8 @@ github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJw github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.1.5 h1:zqZvIKAdSC62e5jv5UDgBWyP61KrLHxxUXjH/MAJlB4= -github.com/projectdiscovery/goflags v0.1.5/go.mod h1:mms5rLXW0+jjbFFTyl/4t9VwtP5TCE+1Y1pReezluAI= +github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhNgdGvfg+Z4= +github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= github.com/projectdiscovery/gologger v1.1.5 h1:CT2Jrpzusca+dwNREMMz4RnAreJ0RqaISRU4SBcZC+M= github.com/projectdiscovery/gologger v1.1.5/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/hmap v0.0.3 h1:7v1cltUmfbOySZBopSOWlkGyVyW9ftxofY09Bz+lctk= @@ -401,8 +401,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.0.0 h1:krS0QRbh4wHqeOybVQwBImY9PmIqzFOYM9Q+cKlEiZE= github.com/projectdiscovery/tlsx v1.0.0/go.mod h1:LOgQpCTE96d/18+Zwu8p5b8ujOGPn6lIDqMIQCJHoXM= -github.com/projectdiscovery/utils v0.0.3 h1:pAjZTGYpnATRc6uaNACdiHv4joZ0Ml7Wpu0dudpcGfM= -github.com/projectdiscovery/utils v0.0.3/go.mod h1:ne3eSlZlUKuhjHr8FfsfGcGteCzxcbJvFBx4VDBCxK0= +github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 h1:/fZvw6gT1fzdmMLMBBw75OrJ0Z6g7dulQrxM9FRp1qU= +github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/projectdiscovery/wappalyzergo v0.0.73 h1:+ctoMja3kmALX1MDcpDsoqY3cU83hd3/fBSi7dm8y5M= github.com/projectdiscovery/wappalyzergo v0.0.73/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -431,9 +431,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= From 20c3ee061e423c80517812726f1d36d9e916603d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Dec 2022 03:06:19 +0530 Subject: [PATCH 341/355] chore(deps): bump github.com/projectdiscovery/ratelimit from 0.0.2 to 0.0.4 (#914) chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.2 to 0.0.4. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.2...v0.0.4) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index e5dd954..db7bfe6 100644 --- a/go.mod +++ b/go.mod @@ -49,12 +49,12 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/asnmap v0.0.1 github.com/projectdiscovery/dsl v0.0.3 - github.com/projectdiscovery/ratelimit v0.0.2 + github.com/projectdiscovery/ratelimit v0.0.4 github.com/projectdiscovery/tlsx v1.0.0 github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.9.0 - golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 + golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 ) require ( diff --git a/go.sum b/go.sum index 884f8af..b9d620a 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgy github.com/projectdiscovery/mapcidr v1.0.3/go.mod h1:/0lEXlu/q0t5u34vIVF6odHR+JCdD3CIHNsMXo7nwrU= github.com/projectdiscovery/networkpolicy v0.0.3 h1:OZFPkMVY6SJxc1ncuRXB2VlT6xlzJl5eWDkvpWn/GMM= github.com/projectdiscovery/networkpolicy v0.0.3/go.mod h1:DIXwKs3sQyfCoWHKRLQiRrEorSQW4Zrh4ftu7oDVK6w= -github.com/projectdiscovery/ratelimit v0.0.2 h1:vRscyCcmTWndd16Ed3lC51196eKYena+xpYfrDomjMg= -github.com/projectdiscovery/ratelimit v0.0.2/go.mod h1:WBz8N1P+CyxnfUoGfVCqah4NZ2SreSX7v9dY8wIlK70= +github.com/projectdiscovery/ratelimit v0.0.4 h1:2TOAhuOUMlKrzwissru2mFnSd8eg2WddIQKcAyYEkGs= +github.com/projectdiscovery/ratelimit v0.0.4/go.mod h1:QK9+yt3ArGWINdj6unGjehtJA/NdlAiF59gaj2FtfEs= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f h1:HR3R/nhELwLXufUlO1ZkKVqrZl4lN1cWFBdN8RcMuLo= @@ -567,8 +567,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= -golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= +golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From 8700f133321261e065d39f394912cd278a772723 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Dec 2022 03:07:05 +0530 Subject: [PATCH 342/355] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.3 to 0.0.4 (#916) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.3 to 0.0.4. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.3...v0.0.4) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index db7bfe6..e97b3ef 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.6 github.com/projectdiscovery/gologger v1.1.5 - github.com/projectdiscovery/hmap v0.0.3 + github.com/projectdiscovery/hmap v0.0.4 github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e diff --git a/go.sum b/go.sum index b9d620a..e4855f1 100644 --- a/go.sum +++ b/go.sum @@ -376,8 +376,8 @@ github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhN github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= github.com/projectdiscovery/gologger v1.1.5 h1:CT2Jrpzusca+dwNREMMz4RnAreJ0RqaISRU4SBcZC+M= github.com/projectdiscovery/gologger v1.1.5/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= -github.com/projectdiscovery/hmap v0.0.3 h1:7v1cltUmfbOySZBopSOWlkGyVyW9ftxofY09Bz+lctk= -github.com/projectdiscovery/hmap v0.0.3/go.mod h1:uYaZSMeXPu5QjDInvtGd/KxTnvF709a0LroIb+FEh1E= +github.com/projectdiscovery/hmap v0.0.4 h1:POJURb6YmeA9pBiKN1DqgVQO9TI2Y9aBFYEd0WikXjQ= +github.com/projectdiscovery/hmap v0.0.4/go.mod h1:uYaZSMeXPu5QjDInvtGd/KxTnvF709a0LroIb+FEh1E= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= From 701dc63568bf0d713d39e36495e48395b1d18252 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Dec 2022 03:07:31 +0530 Subject: [PATCH 343/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.73 to 0.0.75 (#915) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.73 to 0.0.75. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.73...v0.0.75) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e97b3ef..34f4133 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.1 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/projectdiscovery/wappalyzergo v0.0.73 + github.com/projectdiscovery/wappalyzergo v0.0.75 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index e4855f1..0cb104c 100644 --- a/go.sum +++ b/go.sum @@ -403,8 +403,8 @@ github.com/projectdiscovery/tlsx v1.0.0 h1:krS0QRbh4wHqeOybVQwBImY9PmIqzFOYM9Q+c github.com/projectdiscovery/tlsx v1.0.0/go.mod h1:LOgQpCTE96d/18+Zwu8p5b8ujOGPn6lIDqMIQCJHoXM= github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 h1:/fZvw6gT1fzdmMLMBBw75OrJ0Z6g7dulQrxM9FRp1qU= github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= -github.com/projectdiscovery/wappalyzergo v0.0.73 h1:+ctoMja3kmALX1MDcpDsoqY3cU83hd3/fBSi7dm8y5M= -github.com/projectdiscovery/wappalyzergo v0.0.73/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/wappalyzergo v0.0.75 h1:IRiI4CtMpDfjMDa1FmWinEQsGOk4tqNkr+PLzhvXW18= +github.com/projectdiscovery/wappalyzergo v0.0.75/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 88c49b812b2b3e6f0f34de48c1a9eefe8eb445f2 Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Wed, 28 Dec 2022 19:10:22 +0100 Subject: [PATCH 344/355] added ztls flag (#896) * added ztls to httpx * adding missing dialtls * updated readme.md * add ztls functional test Co-authored-by: mzack Co-authored-by: Mzack9999 Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- README.md | 23 +++++++++++++---------- cmd/functional-test/testcases.txt | 1 + common/httpx/httpx.go | 3 ++- common/httpx/option.go | 1 + go.mod | 8 ++++---- go.sum | 11 +++++------ runner/options.go | 2 ++ runner/runner.go | 1 + 8 files changed, 29 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index a936317..b4c8cf8 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ MATCHERS: -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string match response with specified string (-ms admin) -mr, -match-regex string match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (fastly, incapsula, oracle, google, azure, cloudflare, cloudfront, sucuri, leaseweb, akamai) + -mcdn, -match-cdn string[] match host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') -mdc, -match-condition string match response with dsl expression condition @@ -133,7 +133,7 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) -fs, -filter-string string filter response with specified string (-fs admin) -fe, -filter-regex string filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (fastly, incapsula, oracle, google, azure, cloudflare, cloudfront, sucuri, leaseweb, akamai) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') -fdc, -filter-condition string filter response with dsl expression condition @@ -155,14 +155,16 @@ MISCELLANEOUS: -ldv, -list-dsl-variables list json output field keys name that support dsl matcher/filter OUTPUT: - -o, -output string file to write output results - -sr, -store-response store http response to output directory - -srd, -store-response-dir string store http response to custom directory - -csv store output in csv format - -json store output in JSONL(ines) format - -irr, -include-response include http request/response in JSON output (-json only) - -include-chain include redirect http chain in JSON output (-json only) - -store-chain include http redirect chain in responses (-sr only) + -o, -output string file to write output results + -sr, -store-response store http response to output directory + -srd, -store-response-dir string store http response to custom directory + -csv store output in csv format + -csvo, -csv-output-encoding string define output encoding + -json store output in JSONL(ines) format + -irr, -include-response include http request/response in JSON output (-json only) + -irrb, -include-response-base64 include base64 encoded http request/response in JSON output (-json only) + -include-chain include redirect http chain in JSON output (-json only) + -store-chain include http redirect chain in responses (-sr only) CONFIGURATIONS: -r, -resolvers string[] list of custom resolver (file or comma separated) @@ -183,6 +185,7 @@ CONFIGURATIONS: -s, -stream stream mode - start elaborating input targets without sorting -sd, -skip-dedupe disable dedupe input items (only used with stream mode) -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443 + -ztls use ztls library with autofallback to standard one for tls13 DEBUG: -health-check, -hc run diagnostic check up diff --git a/cmd/functional-test/testcases.txt b/cmd/functional-test/testcases.txt index 0121c7c..df81414 100644 --- a/cmd/functional-test/testcases.txt +++ b/cmd/functional-test/testcases.txt @@ -15,4 +15,5 @@ scanme.sh {{binary}} -silent -x all scanme.sh {{binary}} -silent -body 'a=b' scanme.sh {{binary}} -silent -exclude-cdn scanme.sh {{binary}} -silent -ports https:443 +scanme.sh {{binary}} -silent -ztls https://scanme.sh?a=1*1 {{binary}} -silent \ No newline at end of file diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 56041df..0d0d952 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -43,6 +43,7 @@ func New(options *Options) (*HTTPX, error) { fastdialerOpts.Deny = options.Deny fastdialerOpts.Allow = options.Allow fastdialerOpts.WithDialerHistory = true + fastdialerOpts.WithZTLS = options.ZTLS if len(options.Resolvers) > 0 { fastdialerOpts.BaseResolvers = options.Resolvers } @@ -102,9 +103,9 @@ func New(options *Options) (*HTTPX, error) { return nil } } - transport := &http.Transport{ DialContext: httpx.Dialer.Dial, + DialTLSContext: httpx.Dialer.DialTLS, MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, diff --git a/common/httpx/option.go b/common/httpx/option.go index b703bd5..bec97ed 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -27,6 +27,7 @@ type Options struct { MaxRedirects int Unsafe bool TLSGrab bool + ZTLS bool // VHOSTs options VHostIgnoreStatusCode bool VHostIgnoreContentLength bool diff --git a/go.mod b/go.mod index 34f4133..fc55812 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/pkg/errors v0.9.1 github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.9 - github.com/projectdiscovery/fastdialer v0.0.19 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c @@ -34,9 +33,9 @@ require ( github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/net v0.4.0 - golang.org/x/sys v0.3.0 // indirect - golang.org/x/text v0.5.0 + golang.org/x/net v0.2.0 + golang.org/x/sys v0.2.0 // indirect + golang.org/x/text v0.4.0 ) require github.com/spaolacci/murmur3 v1.1.0 @@ -49,6 +48,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/projectdiscovery/asnmap v0.0.1 github.com/projectdiscovery/dsl v0.0.3 + github.com/projectdiscovery/fastdialer v0.0.19 github.com/projectdiscovery/ratelimit v0.0.4 github.com/projectdiscovery/tlsx v1.0.0 github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 diff --git a/go.sum b/go.sum index 0cb104c..9a3fc14 100644 --- a/go.sum +++ b/go.sum @@ -639,8 +639,8 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -717,8 +717,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -730,9 +730,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/runner/options.go b/runner/options.go index c3db6fd..09fe4c7 100644 --- a/runner/options.go +++ b/runner/options.go @@ -235,6 +235,7 @@ type Options struct { OutputFilterFavicon goflags.StringSlice OutputMatchFavicon goflags.StringSlice LeaveDefaultPorts bool + ZTLS bool OutputLinesCount bool OutputMatchLinesCount string matchLinesCount []int @@ -379,6 +380,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.Stream, "stream", "s", false, "stream mode - start elaborating input targets without sorting"), flagSet.BoolVarP(&options.SkipDedupe, "skip-dedupe", "sd", false, "disable dedupe input items (only used with stream mode)"), flagSet.BoolVarP(&options.LeaveDefaultPorts, "leave-default-ports", "ldp", false, "leave default http/https ports in host header (eg. http://host:80 - https//host:443"), + flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13"), ) flagSet.CreateGroup("debug", "Debug", diff --git a/runner/runner.go b/runner/runner.go index 6258ed7..c693cdb 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -127,6 +127,7 @@ func New(options *Options) (*Runner, error) { } httpxOptions.Deny = options.Deny httpxOptions.Allow = options.Allow + httpxOptions.ZTLS = options.ZTLS httpxOptions.MaxResponseBodySizeToSave = int64(options.MaxResponseBodySizeToSave) httpxOptions.MaxResponseBodySizeToRead = int64(options.MaxResponseBodySizeToRead) // adjust response size saved according to the max one read by the server From c0f2a9afe028f7e382a66315648953750345405b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 16:03:10 +0530 Subject: [PATCH 345/355] chore(deps): bump golang.org/x/net from 0.2.0 to 0.4.0 (#917) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.2.0 to 0.4.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.2.0...v0.4.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index fc55812..f0d3732 100644 --- a/go.mod +++ b/go.mod @@ -33,9 +33,9 @@ require ( github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/net v0.2.0 - golang.org/x/sys v0.2.0 // indirect - golang.org/x/text v0.4.0 + golang.org/x/net v0.4.0 + golang.org/x/sys v0.3.0 // indirect + golang.org/x/text v0.5.0 ) require github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index 9a3fc14..0cb104c 100644 --- a/go.sum +++ b/go.sum @@ -639,8 +639,8 @@ golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -717,8 +717,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -730,8 +730,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From e6404ba6ec93f7417da4cee6c15150ee086a4981 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 16:03:20 +0530 Subject: [PATCH 346/355] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.4 to 0.0.5 (#918) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.4 to 0.0.5. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.4...v0.0.5) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f0d3732..72cee74 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.6 github.com/projectdiscovery/gologger v1.1.5 - github.com/projectdiscovery/hmap v0.0.4 + github.com/projectdiscovery/hmap v0.0.5 github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e diff --git a/go.sum b/go.sum index 0cb104c..a71c7d3 100644 --- a/go.sum +++ b/go.sum @@ -376,8 +376,8 @@ github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhN github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= github.com/projectdiscovery/gologger v1.1.5 h1:CT2Jrpzusca+dwNREMMz4RnAreJ0RqaISRU4SBcZC+M= github.com/projectdiscovery/gologger v1.1.5/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= -github.com/projectdiscovery/hmap v0.0.4 h1:POJURb6YmeA9pBiKN1DqgVQO9TI2Y9aBFYEd0WikXjQ= -github.com/projectdiscovery/hmap v0.0.4/go.mod h1:uYaZSMeXPu5QjDInvtGd/KxTnvF709a0LroIb+FEh1E= +github.com/projectdiscovery/hmap v0.0.5 h1:E3ZFqoEtP+iHQWdby8SDrpEt9JTYx+585TBmoPyY+Gg= +github.com/projectdiscovery/hmap v0.0.5/go.mod h1:JC110Qbr2WzSeLFMj6qMB5ig04Qh1bTsO2ucK6eXYmg= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= From 0460e1538c54c0fdf58b731cc6ac3ab238620358 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 16:03:30 +0530 Subject: [PATCH 347/355] chore(deps): bump github.com/projectdiscovery/wappalyzergo from 0.0.75 to 0.0.76 (#919) chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.0.75 to 0.0.76. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.0.75...v0.0.76) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 72cee74..d9fcb5a 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.1 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/projectdiscovery/wappalyzergo v0.0.75 + github.com/projectdiscovery/wappalyzergo v0.0.76 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect diff --git a/go.sum b/go.sum index a71c7d3..7d43eba 100644 --- a/go.sum +++ b/go.sum @@ -403,8 +403,8 @@ github.com/projectdiscovery/tlsx v1.0.0 h1:krS0QRbh4wHqeOybVQwBImY9PmIqzFOYM9Q+c github.com/projectdiscovery/tlsx v1.0.0/go.mod h1:LOgQpCTE96d/18+Zwu8p5b8ujOGPn6lIDqMIQCJHoXM= github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 h1:/fZvw6gT1fzdmMLMBBw75OrJ0Z6g7dulQrxM9FRp1qU= github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= -github.com/projectdiscovery/wappalyzergo v0.0.75 h1:IRiI4CtMpDfjMDa1FmWinEQsGOk4tqNkr+PLzhvXW18= -github.com/projectdiscovery/wappalyzergo v0.0.75/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/wappalyzergo v0.0.76 h1:aG15xPhVY5sK/o3GlGiHrGLpmIkDSUmpbLTGnjVpeAc= +github.com/projectdiscovery/wappalyzergo v0.0.76/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From a31018c76a2abf5b4c441d1780b14a6745e510f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 15:52:30 +0530 Subject: [PATCH 348/355] chore(deps): bump github.com/projectdiscovery/tlsx from 1.0.0 to 1.0.3 (#921) Bumps [github.com/projectdiscovery/tlsx](https://github.com/projectdiscovery/tlsx) from 1.0.0 to 1.0.3. - [Release notes](https://github.com/projectdiscovery/tlsx/releases) - [Changelog](https://github.com/projectdiscovery/tlsx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/tlsx/compare/v1.0.0...v1.0.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/tlsx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d9fcb5a..e11540a 100644 --- a/go.mod +++ b/go.mod @@ -50,8 +50,8 @@ require ( github.com/projectdiscovery/dsl v0.0.3 github.com/projectdiscovery/fastdialer v0.0.19 github.com/projectdiscovery/ratelimit v0.0.4 - github.com/projectdiscovery/tlsx v1.0.0 - github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 + github.com/projectdiscovery/tlsx v1.0.3 + github.com/projectdiscovery/utils v0.0.4-0.20230102120019-c7a04e2045be github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.9.0 golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 diff --git a/go.sum b/go.sum index 7d43eba..e8f7e83 100644 --- a/go.sum +++ b/go.sum @@ -399,10 +399,10 @@ github.com/projectdiscovery/sliceutil v0.0.1 h1:YoCqCMcdwz+gqNfW5hFY8UvNHoA6SfyB github.com/projectdiscovery/sliceutil v0.0.1/go.mod h1:0wBmhU5uTDwMfrEZfvwH9qa5k60Q4shPVOC9E6LGsDI= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.0.0 h1:krS0QRbh4wHqeOybVQwBImY9PmIqzFOYM9Q+cKlEiZE= -github.com/projectdiscovery/tlsx v1.0.0/go.mod h1:LOgQpCTE96d/18+Zwu8p5b8ujOGPn6lIDqMIQCJHoXM= -github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54 h1:/fZvw6gT1fzdmMLMBBw75OrJ0Z6g7dulQrxM9FRp1qU= -github.com/projectdiscovery/utils v0.0.4-0.20221214110533-9f95ee986a54/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= +github.com/projectdiscovery/tlsx v1.0.3 h1:D1g+Ji6fQj4rxxABqi6Q/S7oiY3Cm52uOqeyOVSN8Hs= +github.com/projectdiscovery/tlsx v1.0.3/go.mod h1:JDGqITCgTfL6pyu+nzeFOjuGUzj5MPiT9OBw634G/pM= +github.com/projectdiscovery/utils v0.0.4-0.20230102120019-c7a04e2045be h1:H65vXvA5EasyvlgjFd24cdHKN7Tp/6/1MhVU0UpAMg4= +github.com/projectdiscovery/utils v0.0.4-0.20230102120019-c7a04e2045be/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/projectdiscovery/wappalyzergo v0.0.76 h1:aG15xPhVY5sK/o3GlGiHrGLpmIkDSUmpbLTGnjVpeAc= github.com/projectdiscovery/wappalyzergo v0.0.76/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= From 50391dfc7ab4f7a6dc5971179961f6254fa0e455 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jan 2023 22:18:47 +0530 Subject: [PATCH 349/355] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.5 to 0.0.6 (#920) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.5 to 0.0.6. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.5...v0.0.6) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 35 +++-- go.sum | 439 +++++++++++++-------------------------------------------- 2 files changed, 112 insertions(+), 362 deletions(-) diff --git a/go.mod b/go.mod index e11540a..0c3aa93 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.6 github.com/projectdiscovery/gologger v1.1.5 - github.com/projectdiscovery/hmap v0.0.5 + github.com/projectdiscovery/hmap v0.0.6 github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/mapcidr v1.0.3 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e @@ -30,7 +30,6 @@ require ( github.com/projectdiscovery/wappalyzergo v0.0.76 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 - github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.10.0 // indirect golang.org/x/net v0.4.0 @@ -54,35 +53,35 @@ require ( github.com/projectdiscovery/utils v0.0.4-0.20230102120019-c7a04e2045be github.com/stretchr/testify v1.8.1 go.uber.org/multierr v1.9.0 - golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 + golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 ) require ( - github.com/DataDog/zstd v1.4.5 // indirect + github.com/DataDog/zstd v1.5.2 // indirect github.com/Knetic/govaluate v3.0.0+incompatible // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/cockroachdb/errors v1.8.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect - github.com/cockroachdb/pebble v0.0.0-20221115214559-95662c240d17 // indirect - github.com/cockroachdb/redact v1.0.8 // indirect - github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect + github.com/cockroachdb/errors v1.9.0 // indirect + github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect + github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b // indirect + github.com/cockroachdb/redact v1.1.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dsnet/compress v0.0.1 // indirect + github.com/getsentry/sentry-go v0.16.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.11.13 // indirect + github.com/klauspost/compress v1.15.13 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -93,10 +92,10 @@ require ( github.com/projectdiscovery/networkpolicy v0.0.3 // indirect github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect github.com/projectdiscovery/retryabledns v1.0.17 // indirect - github.com/prometheus/client_golang v1.12.0 // indirect - github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_golang v1.14.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/syndtr/goleveldb v1.0.0 // indirect @@ -119,8 +118,8 @@ require ( golang.org/x/crypto v0.1.0 // indirect golang.org/x/mod v0.6.0 // indirect golang.org/x/tools v0.2.0 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e8f7e83..9d3587d 100644 --- a/go.sum +++ b/go.sum @@ -1,43 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= -github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= @@ -51,11 +20,6 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -64,8 +28,6 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMfqcK8p0g= @@ -75,27 +37,27 @@ github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0 github.com/bxcodec/faker/v4 v4.0.0-beta.3 h1:gqYNBvN72QtzKkYohNDKQlm+pg+uwBDVMN28nWHS18k= github.com/bxcodec/faker/v4 v4.0.0-beta.3/go.mod h1:m6+Ch1Lj3fqW/unZmvkXIdxWS5+XQWPWxcbbQW2X+Ho= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= -github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/datadriven v1.0.1-0.20211007161720-b558070c3be0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/datadriven v1.0.1-0.20220214170620-9913f5bc19b7/go.mod h1:hi0MtSY3AYDQNDi83kDkMH5/yqM/CsIrsOITkSoH7KI= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= -github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= -github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/errors v1.8.8/go.mod h1:z6VnEL3hZ/2ONZEvG7S5Ym0bU2AqPcEKnIiA1wbsSu0= +github.com/cockroachdb/errors v1.9.0 h1:B48dYem5SlAY7iU8AKsgedb4gH6mo+bDkbtLIvM/a88= +github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2OYJBK4GBNYQ= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20221115214559-95662c240d17 h1:D2i3P62ObgXAR0fC2j0qi2/hFnJv/MBpINAPiRqIN8I= -github.com/cockroachdb/pebble v0.0.0-20221115214559-95662c240d17/go.mod h1:qf9bLis2yy1XyNYD01wvIHPabuC1STzQsvGibYVsom4= -github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= -github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b h1:6TtBvyITBhpkL8EvEEZdS9om8egSxP/mUML01aM3pyY= +github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b/go.mod h1:JsehdjcR1QgLZkqBeYrbVdE3cdxbdrycA/PN+Cg+RNw= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -121,6 +83,7 @@ github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZi github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= @@ -132,49 +95,33 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= +github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= +github.com/getsentry/sentry-go v0.16.0 h1:owk+S+5XcgJLlGR/3+3s6N4d+uKwqYvh/eS0AIMjPWo= +github.com/getsentry/sentry-go v0.16.0/go.mod h1:ZXCloQLj0pG7mja5NK6NPf2V4A88YJ4pNlc2mOHwh6Y= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -183,7 +130,6 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -192,42 +138,24 @@ github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= @@ -236,49 +164,50 @@ github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= +github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= +github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= +github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= -github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= +github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -286,20 +215,25 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= +github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= @@ -323,10 +257,11 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= @@ -334,6 +269,7 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= @@ -348,7 +284,6 @@ github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -376,8 +311,8 @@ github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhN github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= github.com/projectdiscovery/gologger v1.1.5 h1:CT2Jrpzusca+dwNREMMz4RnAreJ0RqaISRU4SBcZC+M= github.com/projectdiscovery/gologger v1.1.5/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= -github.com/projectdiscovery/hmap v0.0.5 h1:E3ZFqoEtP+iHQWdby8SDrpEt9JTYx+585TBmoPyY+Gg= -github.com/projectdiscovery/hmap v0.0.5/go.mod h1:JC110Qbr2WzSeLFMj6qMB5ig04Qh1bTsO2ucK6eXYmg= +github.com/projectdiscovery/hmap v0.0.6 h1:nwF553wjvDdFF6QtlVfB80koojn8WfJGOQvkW0dzDuk= +github.com/projectdiscovery/hmap v0.0.6/go.mod h1:xXSxFWk/zyiBrreQqgLTEnGYhufafVcCSXESBnqKOBU= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= github.com/projectdiscovery/mapcidr v1.0.3 h1:SGtOOEz0AxthVO7ZonMvhrJ/AQkHIXCVgyZqJdY0cAY= @@ -405,32 +340,19 @@ github.com/projectdiscovery/utils v0.0.4-0.20230102120019-c7a04e2045be h1:H65vXv github.com/projectdiscovery/utils v0.0.4-0.20230102120019-c7a04e2045be/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/projectdiscovery/wappalyzergo v0.0.76 h1:aG15xPhVY5sK/o3GlGiHrGLpmIkDSUmpbLTGnjVpeAc= github.com/projectdiscovery/wappalyzergo v0.0.76/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg= -github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y= -github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= @@ -439,18 +361,14 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= -github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -467,6 +385,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -494,7 +413,9 @@ github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSR github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -504,6 +425,7 @@ github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKn github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 h1:oRCu5zb6sklsDvy5sOz3dFqGg5vAEYBBD2MAYhNThCQ= @@ -521,9 +443,7 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/ github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -535,11 +455,6 @@ github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be h1:M5QjuCbUeNZsup53dl github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -547,46 +462,24 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w= -golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws= +golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -596,57 +489,34 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -654,65 +524,41 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -723,66 +569,31 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= @@ -791,71 +602,20 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -864,13 +624,11 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -882,29 +640,22 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From f7dc786dc11c23c740920c402974ca0cc6619563 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Jan 2023 15:50:07 +0530 Subject: [PATCH 350/355] chore(deps): bump golang.org/x/text from 0.5.0 to 0.6.0 (#923) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/golang/text/releases) - [Commits](https://github.com/golang/text/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: golang.org/x/text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0c3aa93..b7ac7cd 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect golang.org/x/net v0.4.0 golang.org/x/sys v0.3.0 // indirect - golang.org/x/text v0.5.0 + golang.org/x/text v0.6.0 ) require github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index 9d3587d..83e24f3 100644 --- a/go.sum +++ b/go.sum @@ -576,8 +576,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From e3d63a4a485e37f75f2e05fea6d0a8b7e0ed35c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Jan 2023 20:41:11 +0530 Subject: [PATCH 351/355] chore(deps): bump golang.org/x/net from 0.4.0 to 0.5.0 (#924) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.4.0 to 0.5.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b7ac7cd..f698cf9 100644 --- a/go.mod +++ b/go.mod @@ -32,8 +32,8 @@ require ( github.com/rs/xid v1.4.0 go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/net v0.4.0 - golang.org/x/sys v0.3.0 // indirect + golang.org/x/net v0.5.0 + golang.org/x/sys v0.4.0 // indirect golang.org/x/text v0.6.0 ) diff --git a/go.sum b/go.sum index 83e24f3..44ebefd 100644 --- a/go.sum +++ b/go.sum @@ -514,8 +514,8 @@ golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -563,8 +563,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From d37f3924fac194c20188c153a19b5787d01c5838 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jan 2023 16:41:58 +0530 Subject: [PATCH 352/355] chore(deps): bump github.com/projectdiscovery/gologger from 1.1.5 to 1.1.6 (#925) * chore(deps): bump github.com/projectdiscovery/gologger Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.5 to 1.1.6. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.5...v1.1.6) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * bumping gologger version Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mzack9999 --- go.mod | 4 ++-- go.sum | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index f698cf9..56289c7 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/projectdiscovery/fileutil v0.0.3 // indirect github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c github.com/projectdiscovery/goflags v0.1.6 - github.com/projectdiscovery/gologger v1.1.5 + github.com/projectdiscovery/gologger v1.1.7 github.com/projectdiscovery/hmap v0.0.6 github.com/projectdiscovery/iputil v0.0.2 // indirect github.com/projectdiscovery/mapcidr v1.0.3 @@ -107,7 +107,7 @@ require ( github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect - github.com/ulikunitz/xz v0.5.7 // indirect + github.com/ulikunitz/xz v0.5.11 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect diff --git a/go.sum b/go.sum index 44ebefd..e8d3bb3 100644 --- a/go.sum +++ b/go.sum @@ -309,8 +309,8 @@ github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= github.com/projectdiscovery/goflags v0.1.6 h1:EXigzX4lJmn/fLMnULdc03O7WW+DjiYZhNgdGvfg+Z4= github.com/projectdiscovery/goflags v0.1.6/go.mod h1:yILgA7gbrHuTpIvMfikbivzoxkyxBD1Y5/PRHiGTIFk= -github.com/projectdiscovery/gologger v1.1.5 h1:CT2Jrpzusca+dwNREMMz4RnAreJ0RqaISRU4SBcZC+M= -github.com/projectdiscovery/gologger v1.1.5/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= +github.com/projectdiscovery/gologger v1.1.7 h1:QeByO4NR8vlZ5ZM2XIOJARRj5WOOU5Ix35FBQ1iH5Rk= +github.com/projectdiscovery/gologger v1.1.7/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= github.com/projectdiscovery/hmap v0.0.6 h1:nwF553wjvDdFF6QtlVfB80koojn8WfJGOQvkW0dzDuk= github.com/projectdiscovery/hmap v0.0.6/go.mod h1:xXSxFWk/zyiBrreQqgLTEnGYhufafVcCSXESBnqKOBU= github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= @@ -417,8 +417,9 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= From db1bc2ebba1ea1c49bb713430429c42687f99831 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 6 Jan 2023 13:58:08 +0100 Subject: [PATCH 353/355] Reworking favicon hash calculation (#906) * Reworking favicon hash calculation * removing debug code * bumping go to 1.19 * removing automatic follow redirect * more edge cases * added favicon path info --- .github/workflows/build-test.yml | 2 +- .github/workflows/functional-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release-binary.yml | 2 +- .github/workflows/sonarcloud.yml | 2 +- README.md | 2 +- common/stringz/stringz.go | 18 +++++- go.mod | 2 +- runner/runner.go | 83 +++++++++++++++++++++++++-- runner/types.go | 1 + 10 files changed, 103 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 54800ad..2cc80ed 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 32e40cc..5199b5e 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 2e99e68..0a1ad15 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -12,7 +12,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 98365c7..77359fd 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -18,7 +18,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v4 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 3740833..611a120 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -16,7 +16,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Run unit Tests run: | diff --git a/README.md b/README.md index b4c8cf8..7bc54a2 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ # Installation Instructions -`httpx` requires **go1.18** to install successfully. Run the following command to get the repo: +`httpx` requires **go1.19** to install successfully. Run the following command to get the repo: ```sh go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go index d463804..e9ad5b6 100644 --- a/common/stringz/stringz.go +++ b/common/stringz/stringz.go @@ -3,10 +3,13 @@ package stringz import ( "bytes" "encoding/base64" + "errors" + "net/http" "net/url" "strconv" "strings" + stringsutil "github.com/projectdiscovery/utils/strings" urlutil "github.com/projectdiscovery/utils/url" "github.com/spaolacci/murmur3" ) @@ -106,7 +109,12 @@ func GetInvalidURI(rawURL string) (bool, string) { return false, "" } -func FaviconHash(data []byte) int32 { +func isContentTypeImage(data []byte) bool { + contentType := http.DetectContentType(data) + return stringsutil.HasPrefixAny(contentType, "image/") +} + +func murmurhash(data []byte) int32 { stdBase64 := base64.StdEncoding.EncodeToString(data) stdBase64 = InsertInto(stdBase64, 76, '\n') hasher := murmur3.New32WithSeed(0) @@ -114,6 +122,14 @@ func FaviconHash(data []byte) int32 { return int32(hasher.Sum32()) } +func FaviconHash(data []byte) (int32, error) { + if isContentTypeImage(data) { + return murmurhash(data), nil + } + + return 0, errors.New("content type is not image") +} + func InsertInto(s string, interval int, sep rune) string { var buffer bytes.Buffer before := interval - 1 diff --git a/go.mod b/go.mod index 56289c7..dc50c70 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/httpx -go 1.18 +go 1.19 require ( github.com/akrylysov/pogreb v0.10.1 // indirect diff --git a/runner/runner.go b/runner/runner.go index c693cdb..b3c1552 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -13,6 +13,7 @@ import ( "net/http/httputil" "net/url" "os" + "path" "path/filepath" "reflect" "regexp" @@ -23,6 +24,7 @@ import ( "golang.org/x/exp/maps" + "github.com/PuerkitoBio/goquery" asnmap "github.com/projectdiscovery/asnmap/libs" dsl "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/fastdialer/fastdialer" @@ -1462,11 +1464,12 @@ retry: } builder.WriteRune(']') } - var faviconMMH3 string + + var faviconMMH3, faviconPath string if scanopts.Favicon { - req.URL.Path = "/favicon.ico" - if faviconResp, favErr := hp.Do(req, httpx.UnsafeOptions{}); favErr == nil { - faviconMMH3 = fmt.Sprintf("%d", stringz.FaviconHash(faviconResp.Data)) + var err error + faviconMMH3, faviconPath, err = r.handleFaviconHash(hp, req, resp) + if err == nil { builder.WriteString(" [") if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(faviconMMH3).String()) @@ -1475,9 +1478,10 @@ retry: } builder.WriteRune(']') } else { - gologger.Warning().Msgf("Could not fetch favicon: %s", favErr.Error()) + gologger.Warning().Msgf("could not calculate favicon hash: %s", err) } } + // adding default hashing for json output format if r.options.JSONOutput && len(scanopts.Hashes) == 0 { scanopts.Hashes = "md5,mmh3,sha256,simhash" @@ -1650,6 +1654,7 @@ retry: Technologies: technologies, FinalURL: finalURL, FavIconMMH3: faviconMMH3, + FaviconPath: faviconPath, Hashes: hashesMap, Extracts: extractResult, Jarm: jarmhash, @@ -1665,6 +1670,74 @@ retry: return result } +func (r *Runner) handleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, currentResp *httpx.Response) (string, string, error) { + // Check if current URI is ending with .ico => use current body without additional requests + if path.Ext(req.URL.Path) == ".ico" { + hash, err := r.calculateFaviconHashWithRaw(currentResp.Data) + return hash, req.URL.Path, err + } + + // search in the response of the requested path for element and rel shortcut/mask/apple-touch icon + // link with .ico extension (which will be prioritized if available) + // if not, any of link from other icons can be requested + potentialURLs, err := extractPotentialFavIconsURLs(req, currentResp) + if err != nil { + return "", "", err + } + + // pick the first - we want only one request + if len(potentialURLs) > 0 { + URL, err := url.Parse(potentialURLs[0]) + if err != nil { + return "", "", err + } + if URL.IsAbs() { + req.URL = URL + } else { + if strings.HasPrefix(URL.Path, "/") { + req.URL.Path = URL.Path + } else { + req.URL.Path = "/" + URL.Path + } + } + req.Host = URL.Host + } else { + req.URL = req.URL.JoinPath("favicon.ico") + } + + resp, err := hp.Do(req, httpx.UnsafeOptions{}) + if err != nil { + return "", "", errors.Wrap(err, "could not fetch favicon") + } + hash, err := r.calculateFaviconHashWithRaw(resp.Data) + return hash, req.URL.Path, err +} + +func (r *Runner) calculateFaviconHashWithRaw(data []byte) (string, error) { + hashNum, err := stringz.FaviconHash(data) + if err != nil { + return "", errors.Wrap(err, "could not calculate favicon hash") + } + return fmt.Sprintf("%d", hashNum), nil +} + +func extractPotentialFavIconsURLs(req *retryablehttp.Request, resp *httpx.Response) ([]string, error) { + var potentialURLs []string + document, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data)) + if err != nil { + return nil, err + } + document.Find("link").Each(func(i int, item *goquery.Selection) { + href, okHref := item.Attr("href") + rel, okRel := item.Attr("rel") + isValidRel := okRel && stringsutil.EqualFoldAny(rel, "icon", "shortcut icon", "mask-icon", "apple-touch-icon") + if okHref && isValidRel { + potentialURLs = append(potentialURLs, href) + } + }) + return potentialURLs, nil +} + // SaveResumeConfig to file func (r *Runner) SaveResumeConfig() error { var resumeCfg ResumeCfg diff --git a/runner/types.go b/runner/types.go index c94a6c6..3cf7414 100644 --- a/runner/types.go +++ b/runner/types.go @@ -49,6 +49,7 @@ type Result struct { Host string `json:"host,omitempty" csv:"host"` Path string `json:"path,omitempty" csv:"path"` FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` + FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path"` FinalURL string `json:"final_url,omitempty" csv:"final_url"` ResponseHeader map[string]interface{} `json:"header,omitempty" csv:"header"` RawHeader string `json:"raw_header,omitempty" csv:"raw_header"` From ed50d399b63d6373e107b9f1220496ddbfff2f51 Mon Sep 17 00:00:00 2001 From: Sandeep Singh Date: Fri, 6 Jan 2023 18:34:12 +0530 Subject: [PATCH 354/355] workflow updates (#926) --- .github/workflows/release-binary.yml | 6 ++++-- .goreleaser.yml | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 77359fd..a51fe4e 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -1,9 +1,9 @@ name: 🎉 Release Binary on: - create: + push: tags: - - v* + - '*' workflow_dispatch: jobs: @@ -29,3 +29,5 @@ jobs: env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" + DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 7d5bd2d..78f2618 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -39,4 +39,8 @@ announce: enabled: true channel: '#release' username: GoReleaser - message_template: '{{ .ProjectName }} {{ .Tag }} is out! Check it out at {{ .ReleaseURL }}' \ No newline at end of file + message_template: 'New Release: {{ .ProjectName }} {{ .Tag }} is published! Check it out at {{ .ReleaseURL }}' + + discord: + enabled: true + message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file From 95ca203e09987c9765f90d950548e52c45c18446 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 6 Jan 2023 18:40:14 +0530 Subject: [PATCH 355/355] version update --- runner/banner.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/runner/banner.go b/runner/banner.go index c273f02..63c3209 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,17 +8,14 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.5 + /_/ v1.2.6 ` // Version is the current version of httpx -const Version = `v1.2.5` +const Version = `v1.2.6` // showBanner is used to show the banner to the user func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") - - gologger.Print().Msgf("Use with caution. You are responsible for your actions.\n") - gologger.Print().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") }