From 74e719255035091aaad941a95a18953e1196e89a Mon Sep 17 00:00:00 2001 From: mzack Date: Sun, 8 Aug 2021 22:41:13 +0200 Subject: [PATCH 01/15] begin of work --- v2/go.mod | 6 ++-- v2/go.sum | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 766add0b80..1b0120cbfb 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -18,13 +18,13 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/itchyny/gojq v0.12.4 - github.com/json-iterator/go v1.1.10 + github.com/json-iterator/go v1.1.11 github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mattn/go-runewidth v0.0.10 // indirect - github.com/miekg/dns v1.1.38 + github.com/miekg/dns v1.1.41 github.com/olekukonko/tablewriter v0.0.5 github.com/owenrumney/go-sarif v1.0.4 github.com/pkg/errors v0.9.1 @@ -34,6 +34,7 @@ require ( github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/interactsh v0.0.3 + github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49 // indirect github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.10 github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 @@ -57,7 +58,6 @@ require ( golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df // indirect golang.org/x/net v0.0.0-20210521195947-fe42d452be8f golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 - golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 diff --git a/v2/go.sum b/v2/go.sum index e5fcc17eed..a1d0aeb74d 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -40,8 +40,13 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= +github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -50,10 +55,17 @@ github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDw github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/color v0.0.0-20140713052517-28ad4cc941d6/go.mod h1:k6bCbg1gudUfWem/VvfWXsk5Qqag/NDXzEHu0mNcsPM= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bmkessler/haraka v0.0.0-20180824194238-3cf1081eecd7 h1:MivIyB/CyPzqS6+OHQuWoPwyvBQxTstP040mF/05i6I= +github.com/bmkessler/haraka v0.0.0-20180824194238-3cf1081eecd7/go.mod h1:zduTbYsr7nd/sJWn/q6wJgIrqqNPoVl0HW27Tff9Fa4= +github.com/bproctor/base91 v0.0.0-20160902125316-7de6f1dd34e8 h1:AOcLIabtn5jh31VACsbAI1R6y/FjyFRDyuNxkvHmSTI= +github.com/bproctor/base91 v0.0.0-20160902125316-7de6f1dd34e8/go.mod h1:Qf+Cy/9Oy2JSEKDrf5nBXdsBqEFBNbU8Rzrart5f7h0= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= @@ -62,6 +74,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/anyxml v1.2.2 h1:EqBXr26KEC7tuGLDH7ZSeFmaE3yft0h386m5uydpNbU= +github.com/clbanning/anyxml v1.2.2/go.mod h1:m8+zXuK8aS9lnXzfpSLUUjXoqcZ41osGX+JXr09eOjY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= @@ -70,9 +84,23 @@ github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkb github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cxmcc/tiger v0.0.0-20170524142333-bde35e2713d7 h1:jBEtq1t2gpn2kEzvRlCUxvvrxl5aSWkXNPwe/hwvSNQ= +github.com/cxmcc/tiger v0.0.0-20170524142333-bde35e2713d7/go.mod h1:ruCYvt9rtYymAr4rNmfYJrl1dz8HSXUFP7cufqKOsDI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/blake256 v1.1.0 h1:4AuEhGPT/3TTKFhTfBpZ8hgZE7wJpawcYaEawwsbtqM= +github.com/dchest/blake256 v1.1.0/go.mod h1:xXNWCE1jsAP8DAjP+rKw2MbeqLczjI3TRx2VK+9OEYY= +github.com/dchest/blake2b v1.0.0 h1:KK9LimVmE0MjRl9095XJmKqZ+iLxWATvlcpVFRtaw6s= +github.com/dchest/blake2b v1.0.0/go.mod h1:U034kXgbJpCle2wSk5ybGIVhOSHCVLMDqOzcPEA0F7s= +github.com/dchest/blake2s v1.0.0 h1:gHCBR8ecSImY/Nwk7X0Q2KJAJcpI/HSkUAQDi8MCP4Q= +github.com/dchest/blake2s v1.0.0/go.mod h1:GrKn2Lc4hWqAwRrbneYuvZ6kugiJMrjk3HHtcJkEhbs= +github.com/dchest/blake512 v1.0.0 h1:oDFEQFIqFSeuA34xLtXZ/rWxCXdSjirjzPhey5EUvmA= +github.com/dchest/blake512 v1.0.0/go.mod h1:FV1x7xPPLWukZlpDpWQ88rF/SFwZ5qbskrzhLMB92JI= +github.com/dchest/siphash v1.2.2 h1:9DFz8tQwl9pTVt5iok/9zKyzA1Q6bRGiF3HPiEEVr9I= +github.com/dchest/siphash v1.2.2/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= +github.com/dchest/skein v0.0.0-20171112102903-d7f1022db390 h1:oNcAGoFeaPCgOnlARnJMQqgoq1UMlGwW7PFJddtTF2c= +github.com/dchest/skein v0.0.0-20171112102903-d7f1022db390/go.mod h1:sh8l6PI4IHMaBmo2rlnHxnJDjXY7rxmDeaGSyupxMVM= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= @@ -82,6 +110,8 @@ github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5Jflh github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI= github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo= +github.com/elliotchance/phpserialize v1.3.1 h1:bA32YZF9/WmxJgIFmrweMK03Y74vPAMmRyKTLFs67/Q= +github.com/elliotchance/phpserialize v1.3.1/go.mod h1:gt7XX9+ETUcLXbtTKEuyrqW3lcLUAeS/AnGZ2e49TZs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -147,6 +177,7 @@ github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:od github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -178,8 +209,14 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= +github.com/howeyc/crc16 v0.0.0-20171223171357-2b2a61e366a6 h1:IIVxLyDUYErC950b8kecjoqDet8P5S4lcVRUOM6rdkU= +github.com/howeyc/crc16 v0.0.0-20171223171357-2b2a61e366a6/go.mod h1:JslaLRrzGsOKJgFEPBP65Whn+rdwDQSk0I0MCRFe2Zw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/htruong/go-md2 v0.0.0-20170914203617-c69905b63f6f h1:GcDipGuLklPyATFZtZcwsN4WFCBhQdRsVoJL1vDLp64= +github.com/htruong/go-md2 v0.0.0-20170914203617-c69905b63f6f/go.mod h1:uMYyN0e+BBSemEoe2Ys7jLGs6DIV9zc6Sj8gTHK1XOI= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= @@ -188,14 +225,22 @@ github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921i github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= +github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 h1:4zOlv2my+vf98jT1nQt4bT/yKWUImevYPJ2H344CloE= +github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6/go.mod h1:r/8JmuR0qjuCiEhAolkfvdZgmPiHTnJaG0UXCSeR1Zo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b h1:XUr8tvMEILhphQPp3TFcIudb5KTOzFeD0pJyDn5+5QI= +github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b/go.mod h1:a5Mn24iYVJRUQSkFupGByqykzD+k+wFI8J91zGHuPf8= +github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= +github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= github.com/karlseguin/ccache/v2 v2.0.7 h1:y5Pfi4eiyYCOD6LS/Kj+o6Nb4M5Ngpw9qFQs+v44ZYM= @@ -205,10 +250,14 @@ github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBx github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kisielk/og-rek v1.1.0 h1:u10TvQbPtrlY/6H4+BiFsBywwSVTGFsx0YOVtpx3IbI= +github.com/kisielk/og-rek v1.1.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c= github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -220,23 +269,41 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/martinlindhe/base36 v1.1.0 h1:cIwvvwYse/0+1CkUPYH5ZvVIYG3JrILmQEIbLuar02Y= +github.com/martinlindhe/base36 v1.1.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= +github.com/martinlindhe/bubblebabble v0.0.0-20160819103256-a0a549d8557a h1:qSlkSgMpUT8L21GR0s9VgMNiC0eNYMfcssWdutZNuMw= +github.com/martinlindhe/bubblebabble v0.0.0-20160819103256-a0a549d8557a/go.mod h1:1sdM3rhzqvxbV0Emgfru91xws5iMIKglDdhOMw+jjp0= +github.com/martinlindhe/crc24 v0.0.0-20170911183234-2a4d83245616 h1:tJX0JYmBSy9o9VPtdr1DFjW+AUYdEWmKWA6PM6rWWDU= +github.com/martinlindhe/crc24 v0.0.0-20170911183234-2a4d83245616/go.mod h1:2LSPYSdnKs8BnG5rM9jaR70St2Z0b0LJN7GvCGTHUEA= +github.com/martinlindhe/gogost v0.0.0-20170914195721-31862914ae20 h1:NgvNvoe91W36nQwPNvM8KSyvN7Tgfrp1gGhNqp6iuK8= +github.com/martinlindhe/gogost v0.0.0-20170914195721-31862914ae20/go.mod h1:QWtANgYYeIaHYj7lc8bygiTdXt3t3bQHH6ObdVMIC2s= +github.com/martinlindhe/gohash v0.0.0-20201217001544-b9eda876a50f h1:DVaFMVVMPN5MdgXTfvdlpWZFXGNFIz/71H5QstX8j2s= +github.com/martinlindhe/gohash v0.0.0-20201217001544-b9eda876a50f/go.mod h1:9n5/U1N6G5ig5C/0ParfHS98Bl1dZe/G7TFOgSh/Eec= +github.com/martinlindhe/uu v0.0.0-20171218233747-27d928e7b0bb h1:Xm/Z5Ia8LPjX/j07KNFzNwlLRyBiJSmH7et7cXpjN0w= +github.com/martinlindhe/uu v0.0.0-20171218233747-27d928e7b0bb/go.mod h1:wlz78WxArFqAPmyrsBK+cBo+fpYrMdRM7IZL/o4m4JQ= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mewpkg/hashutil v1.0.1 h1:me5DWIMS8wLeMmtZnMxwNTMIA9nT9HvQZLYVMbmBIyw= +github.com/mewpkg/hashutil v1.0.1/go.mod h1:DSL30IKicBzaNFwGvhg9BG8YkZj0Nb+d6TDPb9P7hQY= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.38 h1:MtIY+fmHUVVgv1AXzmKMWcwdCYxTRPG1EDjpqF4RCEw= github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -257,6 +324,8 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4= github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -265,6 +334,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= +github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd h1:kPBRsSG8ZwNHLOBzDM/vfGi/Oaj/dzdbTeEkv+6UhEw= +github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd/go.mod h1:C8Cevz76yJLH0RpmIa+wWTVnZQyIL6CyhTnJJl6hHas= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew= @@ -276,6 +347,10 @@ github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/interactsh v0.0.3 h1:PUkWk+NzSyd5glMqfORmuqizhsd7c3WdTYBOto/MQIU= github.com/projectdiscovery/interactsh v0.0.3/go.mod h1:dWnKO14d2FLP3kLhI9DecEsiAC/aZiJoUBGFjGhDskY= +github.com/projectdiscovery/mapsutil v0.0.1 h1:hXIQUPvU+5HtPzzyaLvsPg4R/39N77CCAL0ezYhFdgw= +github.com/projectdiscovery/mapsutil v0.0.1/go.mod h1:Q49P7sQzz9m+dWgdx/ryNwOfsClt8ZmHhF4WdXTheyE= +github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49 h1:ktRs9zNmuGovhewcQ1qadnwMEclvPGvUxVSHwJsNjMY= +github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49/go.mod h1:yRk2l4QV4G+biDC4jW6Vuiv9OVMq15V/U072fxH4Gnc= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= @@ -320,6 +395,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tilinna/z85 v1.0.0 h1:uqFnJBlD01dosSeo5sK1G1YGbPuwqVHqR+12OJDRjUw= +github.com/tilinna/z85 v1.0.0/go.mod h1:EfpFU/DUY4ddEy6CRvk2l+UQNEzHbh+bqBQS+04Nkxs= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -369,6 +446,8 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.starlark.net v0.0.0-20210602144842-1cdb82c9e17a h1:wDtSCWGrX9tusypq2Qq9xzaA3Tf/+4D2KaWO+HQvGZE= +go.starlark.net v0.0.0-20210602144842-1cdb82c9e17a/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -385,6 +464,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df h1:y7QZzfUiTwWam+xBn29Ulb8CBwVN5UdzmMDavl9Whlw= golang.org/x/crypto v0.0.0-20210218145215-b8e89b74b9df/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= @@ -448,6 +528,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210502030024-e5908800b52b/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -468,6 +550,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -499,11 +582,15 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -654,6 +741,8 @@ gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/mewpkg/hashutil.v1 v1.0.1 h1:MTPLGywH+y31RBcgee3b4LMkWK6rI1TnWrenlRu6kPw= +gopkg.in/mewpkg/hashutil.v1 v1.0.1/go.mod h1:xPeyz3Qq2Y5ClYcSUywLCyCKHmHpmSOWgrPDbdw3zjQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -662,6 +751,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -671,6 +761,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/blake3 v1.1.4 h1:cVl4fhMGfCaCFrs4sBb8R/iEMeL0g4CfQp/sAHBDxOI= +lukechampine.com/blake3 v1.1.4/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From b92e73b498a439e3d4ab792c1878b61af2855f40 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 9 Aug 2021 19:18:46 +0200 Subject: [PATCH 02/15] replacing govaluate with nebula --- v2/go.mod | 6 +- v2/go.sum | 87 +++++- v2/pkg/operators/common/dsl/dsl.go | 285 ------------------ v2/pkg/operators/matchers/compile.go | 12 - v2/pkg/operators/matchers/match.go | 8 +- v2/pkg/operators/matchers/matchers.go | 3 - .../common/expressions/expressions.go | 30 +- 7 files changed, 98 insertions(+), 333 deletions(-) delete mode 100644 v2/pkg/operators/common/dsl/dsl.go diff --git a/v2/go.mod b/v2/go.mod index a0406a7e64..f014903892 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -2,8 +2,9 @@ module github.com/projectdiscovery/nuclei/v2 go 1.15 +replace github.com/projectdiscovery/nebula => /Users/marcornvh/go/src/github.com/projectdiscovery/nebula + require ( - github.com/Knetic/govaluate v3.0.0+incompatible github.com/andygrunwald/go-jira v1.13.0 github.com/apex/log v1.9.0 github.com/blang/semver v3.5.1+incompatible @@ -22,7 +23,6 @@ require ( github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 - github.com/klauspost/cpuid v1.3.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mattn/go-runewidth v0.0.10 // indirect github.com/miekg/dns v1.1.43 @@ -35,6 +35,7 @@ require ( github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/interactsh v0.0.4 + github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49 github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.10 github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 @@ -44,7 +45,6 @@ require ( github.com/rs/xid v1.3.0 github.com/segmentio/ksuid v1.0.3 github.com/shirou/gopsutil/v3 v3.21.5 - github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.3.1 github.com/stretchr/testify v1.7.0 github.com/syndtr/goleveldb v1.0.0 diff --git a/v2/go.sum b/v2/go.sum index db8a3324e6..0f6d9804c7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -35,15 +35,18 @@ git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+V git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= -github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/andybalholm/brotli v1.0.3 h1:fpcw+r1N1h0Poc1F/pHbW40cUm/lMEQslZtCkBQ0UnM= +github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andygrunwald/go-jira v1.13.0 h1:vvIImGgX32bHfoiyUwkNo+/YrPnRczNarvhLOncP6dE= github.com/andygrunwald/go-jira v1.13.0/go.mod h1:jYi4kFDbRPZTJdJOVJO4mpMMIwdB+rcZwSO58DzPd2I= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -52,10 +55,17 @@ github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDw github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/color v0.0.0-20140713052517-28ad4cc941d6/go.mod h1:k6bCbg1gudUfWem/VvfWXsk5Qqag/NDXzEHu0mNcsPM= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/bmkessler/haraka v0.0.0-20180824194238-3cf1081eecd7 h1:MivIyB/CyPzqS6+OHQuWoPwyvBQxTstP040mF/05i6I= +github.com/bmkessler/haraka v0.0.0-20180824194238-3cf1081eecd7/go.mod h1:zduTbYsr7nd/sJWn/q6wJgIrqqNPoVl0HW27Tff9Fa4= +github.com/bproctor/base91 v0.0.0-20160902125316-7de6f1dd34e8 h1:AOcLIabtn5jh31VACsbAI1R6y/FjyFRDyuNxkvHmSTI= +github.com/bproctor/base91 v0.0.0-20160902125316-7de6f1dd34e8/go.mod h1:Qf+Cy/9Oy2JSEKDrf5nBXdsBqEFBNbU8Rzrart5f7h0= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/c4milo/unpackit v0.1.0 h1:91pWJ6B3svZ4LOE+p3rnyucRK5fZwBdF/yQ/pcZO31I= @@ -64,6 +74,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/anyxml v1.2.2 h1:EqBXr26KEC7tuGLDH7ZSeFmaE3yft0h386m5uydpNbU= +github.com/clbanning/anyxml v1.2.2/go.mod h1:m8+zXuK8aS9lnXzfpSLUUjXoqcZ41osGX+JXr09eOjY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= @@ -72,9 +84,23 @@ github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkb github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cxmcc/tiger v0.0.0-20170524142333-bde35e2713d7 h1:jBEtq1t2gpn2kEzvRlCUxvvrxl5aSWkXNPwe/hwvSNQ= +github.com/cxmcc/tiger v0.0.0-20170524142333-bde35e2713d7/go.mod h1:ruCYvt9rtYymAr4rNmfYJrl1dz8HSXUFP7cufqKOsDI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/blake256 v1.1.0 h1:4AuEhGPT/3TTKFhTfBpZ8hgZE7wJpawcYaEawwsbtqM= +github.com/dchest/blake256 v1.1.0/go.mod h1:xXNWCE1jsAP8DAjP+rKw2MbeqLczjI3TRx2VK+9OEYY= +github.com/dchest/blake2b v1.0.0 h1:KK9LimVmE0MjRl9095XJmKqZ+iLxWATvlcpVFRtaw6s= +github.com/dchest/blake2b v1.0.0/go.mod h1:U034kXgbJpCle2wSk5ybGIVhOSHCVLMDqOzcPEA0F7s= +github.com/dchest/blake2s v1.0.0 h1:gHCBR8ecSImY/Nwk7X0Q2KJAJcpI/HSkUAQDi8MCP4Q= +github.com/dchest/blake2s v1.0.0/go.mod h1:GrKn2Lc4hWqAwRrbneYuvZ6kugiJMrjk3HHtcJkEhbs= +github.com/dchest/blake512 v1.0.0 h1:oDFEQFIqFSeuA34xLtXZ/rWxCXdSjirjzPhey5EUvmA= +github.com/dchest/blake512 v1.0.0/go.mod h1:FV1x7xPPLWukZlpDpWQ88rF/SFwZ5qbskrzhLMB92JI= +github.com/dchest/siphash v1.2.2 h1:9DFz8tQwl9pTVt5iok/9zKyzA1Q6bRGiF3HPiEEVr9I= +github.com/dchest/siphash v1.2.2/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4= +github.com/dchest/skein v0.0.0-20171112102903-d7f1022db390 h1:oNcAGoFeaPCgOnlARnJMQqgoq1UMlGwW7PFJddtTF2c= +github.com/dchest/skein v0.0.0-20171112102903-d7f1022db390/go.mod h1:sh8l6PI4IHMaBmo2rlnHxnJDjXY7rxmDeaGSyupxMVM= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= @@ -84,6 +110,8 @@ github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5Jflh github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/eggsampler/acme/v3 v3.2.1 h1:Lfsrg3M2zt00QRnizOFzdpSfsS9oDvPsGrodXS/w1KI= github.com/eggsampler/acme/v3 v3.2.1/go.mod h1:/qh0rKC/Dh7Jj+p4So7DbWmFNzC4dpcpK53r226Fhuo= +github.com/elliotchance/phpserialize v1.3.1 h1:bA32YZF9/WmxJgIFmrweMK03Y74vPAMmRyKTLFs67/Q= +github.com/elliotchance/phpserialize v1.3.1/go.mod h1:gt7XX9+ETUcLXbtTKEuyrqW3lcLUAeS/AnGZ2e49TZs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -149,6 +177,8 @@ github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:od github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -180,8 +210,14 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hooklift/assert v0.1.0 h1:UZzFxx5dSb9aBtvMHTtnPuvFnBvcEhHTPb9+0+jpEjs= github.com/hooklift/assert v0.1.0/go.mod h1:pfexfvIHnKCdjh6CkkIZv5ic6dQ6aU2jhKghBlXuwwY= +github.com/howeyc/crc16 v0.0.0-20171223171357-2b2a61e366a6 h1:IIVxLyDUYErC950b8kecjoqDet8P5S4lcVRUOM6rdkU= +github.com/howeyc/crc16 v0.0.0-20171223171357-2b2a61e366a6/go.mod h1:JslaLRrzGsOKJgFEPBP65Whn+rdwDQSk0I0MCRFe2Zw= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/htruong/go-md2 v0.0.0-20170914203617-c69905b63f6f h1:GcDipGuLklPyATFZtZcwsN4WFCBhQdRsVoJL1vDLp64= +github.com/htruong/go-md2 v0.0.0-20170914203617-c69905b63f6f/go.mod h1:uMYyN0e+BBSemEoe2Ys7jLGs6DIV9zc6Sj8gTHK1XOI= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= github.com/itchyny/gojq v0.12.4 h1:8zgOZWMejEWCLjbF/1mWY7hY7QEARm7dtuhC6Bp4R8o= @@ -190,6 +226,8 @@ github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921i github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU= github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4= +github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 h1:4zOlv2my+vf98jT1nQt4bT/yKWUImevYPJ2H344CloE= +github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6/go.mod h1:r/8JmuR0qjuCiEhAolkfvdZgmPiHTnJaG0UXCSeR1Zo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -199,6 +237,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b h1:XUr8tvMEILhphQPp3TFcIudb5KTOzFeD0pJyDn5+5QI= +github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b/go.mod h1:a5Mn24iYVJRUQSkFupGByqykzD+k+wFI8J91zGHuPf8= +github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= +github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= @@ -208,6 +250,8 @@ github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBx github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kisielk/og-rek v1.1.0 h1:u10TvQbPtrlY/6H4+BiFsBywwSVTGFsx0YOVtpx3IbI= +github.com/kisielk/og-rek v1.1.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c= github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -224,20 +268,38 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/martinlindhe/base36 v1.1.0 h1:cIwvvwYse/0+1CkUPYH5ZvVIYG3JrILmQEIbLuar02Y= +github.com/martinlindhe/base36 v1.1.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= +github.com/martinlindhe/bubblebabble v0.0.0-20160819103256-a0a549d8557a h1:qSlkSgMpUT8L21GR0s9VgMNiC0eNYMfcssWdutZNuMw= +github.com/martinlindhe/bubblebabble v0.0.0-20160819103256-a0a549d8557a/go.mod h1:1sdM3rhzqvxbV0Emgfru91xws5iMIKglDdhOMw+jjp0= +github.com/martinlindhe/crc24 v0.0.0-20170911183234-2a4d83245616 h1:tJX0JYmBSy9o9VPtdr1DFjW+AUYdEWmKWA6PM6rWWDU= +github.com/martinlindhe/crc24 v0.0.0-20170911183234-2a4d83245616/go.mod h1:2LSPYSdnKs8BnG5rM9jaR70St2Z0b0LJN7GvCGTHUEA= +github.com/martinlindhe/gogost v0.0.0-20170914195721-31862914ae20 h1:NgvNvoe91W36nQwPNvM8KSyvN7Tgfrp1gGhNqp6iuK8= +github.com/martinlindhe/gogost v0.0.0-20170914195721-31862914ae20/go.mod h1:QWtANgYYeIaHYj7lc8bygiTdXt3t3bQHH6ObdVMIC2s= +github.com/martinlindhe/gohash v0.0.0-20201217001544-b9eda876a50f h1:DVaFMVVMPN5MdgXTfvdlpWZFXGNFIz/71H5QstX8j2s= +github.com/martinlindhe/gohash v0.0.0-20201217001544-b9eda876a50f/go.mod h1:9n5/U1N6G5ig5C/0ParfHS98Bl1dZe/G7TFOgSh/Eec= +github.com/martinlindhe/uu v0.0.0-20171218233747-27d928e7b0bb h1:Xm/Z5Ia8LPjX/j07KNFzNwlLRyBiJSmH7et7cXpjN0w= +github.com/martinlindhe/uu v0.0.0-20171218233747-27d928e7b0bb/go.mod h1:wlz78WxArFqAPmyrsBK+cBo+fpYrMdRM7IZL/o4m4JQ= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mewpkg/hashutil v1.0.1 h1:me5DWIMS8wLeMmtZnMxwNTMIA9nT9HvQZLYVMbmBIyw= +github.com/mewpkg/hashutil v1.0.1/go.mod h1:DSL30IKicBzaNFwGvhg9BG8YkZj0Nb+d6TDPb9P7hQY= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -260,6 +322,8 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/owenrumney/go-sarif v1.0.4 h1:0LFC5eHP6amc/9ajM1jDiE52UfXFcl/oozay+X3KgV4= github.com/owenrumney/go-sarif v1.0.4/go.mod h1:DXUGbHwQcCMvqcvZbxh8l/7diHsJVztOKZgmPt88RNI= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -268,6 +332,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= +github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd h1:kPBRsSG8ZwNHLOBzDM/vfGi/Oaj/dzdbTeEkv+6UhEw= +github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd/go.mod h1:C8Cevz76yJLH0RpmIa+wWTVnZQyIL6CyhTnJJl6hHas= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= github.com/projectdiscovery/goflags v0.0.6 h1:4ErduTfSC55cRR3TmUg+TQirBlCuBdBadrluAsy1pew= @@ -278,6 +344,8 @@ github.com/projectdiscovery/hmap v0.0.1 h1:VAONbJw5jP+syI5smhsfkrq9XPGn4aiYy5pR6 github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvzcEI5pJIuShM8= github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= +github.com/projectdiscovery/mapsutil v0.0.1 h1:hXIQUPvU+5HtPzzyaLvsPg4R/39N77CCAL0ezYhFdgw= +github.com/projectdiscovery/mapsutil v0.0.1/go.mod h1:Q49P7sQzz9m+dWgdx/ryNwOfsClt8ZmHhF4WdXTheyE= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= @@ -319,6 +387,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tilinna/z85 v1.0.0 h1:uqFnJBlD01dosSeo5sK1G1YGbPuwqVHqR+12OJDRjUw= +github.com/tilinna/z85 v1.0.0/go.mod h1:EfpFU/DUY4ddEy6CRvk2l+UQNEzHbh+bqBQS+04Nkxs= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -368,6 +438,8 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.starlark.net v0.0.0-20210602144842-1cdb82c9e17a h1:wDtSCWGrX9tusypq2Qq9xzaA3Tf/+4D2KaWO+HQvGZE= +go.starlark.net v0.0.0-20210602144842-1cdb82c9e17a/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -381,6 +453,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -443,6 +517,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210502030024-e5908800b52b/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -493,12 +568,15 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201113233024-12cec1faf1ba/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -649,6 +727,8 @@ gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/mewpkg/hashutil.v1 v1.0.1 h1:MTPLGywH+y31RBcgee3b4LMkWK6rI1TnWrenlRu6kPw= +gopkg.in/mewpkg/hashutil.v1 v1.0.1/go.mod h1:xPeyz3Qq2Y5ClYcSUywLCyCKHmHpmSOWgrPDbdw3zjQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -657,6 +737,7 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -666,6 +747,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/blake3 v1.1.4 h1:cVl4fhMGfCaCFrs4sBb8R/iEMeL0g4CfQp/sAHBDxOI= +lukechampine.com/blake3 v1.1.4/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go deleted file mode 100644 index 8116ef1363..0000000000 --- a/v2/pkg/operators/common/dsl/dsl.go +++ /dev/null @@ -1,285 +0,0 @@ -package dsl - -import ( - "crypto/md5" - "crypto/sha1" - "crypto/sha256" - "encoding/base64" - "encoding/hex" - "errors" - "fmt" - "html" - "math" - "math/rand" - "net/url" - "regexp" - "strings" - "time" - - "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - "github.com/spaolacci/murmur3" -) - -const ( - numbers = "1234567890" - letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" - withCutSetArgsSize = 2 - withBaseRandArgsSize = 3 - withMaxRandArgsSize = withCutSetArgsSize -) - -var functions = map[string]govaluate.ExpressionFunction{ - "len": func(args ...interface{}) (interface{}, error) { - length := len(types.ToString(args[0])) - return float64(length), nil - }, - "toupper": func(args ...interface{}) (interface{}, error) { - return strings.ToUpper(types.ToString(args[0])), nil - }, - "tolower": func(args ...interface{}) (interface{}, error) { - return strings.ToLower(types.ToString(args[0])), nil - }, - "replace": func(args ...interface{}) (interface{}, error) { - return strings.ReplaceAll(types.ToString(args[0]), types.ToString(args[1]), types.ToString(args[2])), nil - }, - "replace_regex": func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(types.ToString(args[1])) - if err != nil { - return nil, err - } - return compiled.ReplaceAllString(types.ToString(args[0]), types.ToString(args[2])), nil - }, - "trim": func(args ...interface{}) (interface{}, error) { - return strings.Trim(types.ToString(args[0]), types.ToString(args[2])), nil - }, - "trimleft": func(args ...interface{}) (interface{}, error) { - return strings.TrimLeft(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "trimright": func(args ...interface{}) (interface{}, error) { - return strings.TrimRight(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "trimspace": func(args ...interface{}) (interface{}, error) { - return strings.TrimSpace(types.ToString(args[0])), nil - }, - "trimprefix": func(args ...interface{}) (interface{}, error) { - return strings.TrimPrefix(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "trimsuffix": func(args ...interface{}) (interface{}, error) { - return strings.TrimSuffix(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "reverse": func(args ...interface{}) (interface{}, error) { - return reverseString(types.ToString(args[0])), nil - }, - // encoding - "base64": func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) - - return sEnc, nil - }, - // python encodes to base64 with lines of 76 bytes terminated by new line "\n" - "base64_py": func(args ...interface{}) (interface{}, error) { - sEnc := base64.StdEncoding.EncodeToString([]byte(types.ToString(args[0]))) - return deserialization.InsertInto(sEnc, 76, '\n'), nil - }, - "base64_decode": func(args ...interface{}) (interface{}, error) { - return base64.StdEncoding.DecodeString(types.ToString(args[0])) - }, - "url_encode": func(args ...interface{}) (interface{}, error) { - return url.PathEscape(types.ToString(args[0])), nil - }, - "url_decode": func(args ...interface{}) (interface{}, error) { - return url.PathUnescape(types.ToString(args[0])) - }, - "hex_encode": func(args ...interface{}) (interface{}, error) { - return hex.EncodeToString([]byte(types.ToString(args[0]))), nil - }, - "hex_decode": func(args ...interface{}) (interface{}, error) { - hx, _ := hex.DecodeString(types.ToString(args[0])) - return string(hx), nil - }, - "html_escape": func(args ...interface{}) (interface{}, error) { - return html.EscapeString(types.ToString(args[0])), nil - }, - "html_unescape": func(args ...interface{}) (interface{}, error) { - return html.UnescapeString(types.ToString(args[0])), nil - }, - // hashing - "md5": func(args ...interface{}) (interface{}, error) { - hash := md5.Sum([]byte(types.ToString(args[0]))) - - return hex.EncodeToString(hash[:]), nil - }, - "sha256": func(args ...interface{}) (interface{}, error) { - h := sha256.New() - _, err := h.Write([]byte(types.ToString(args[0]))) - - if err != nil { - return nil, err - } - return hex.EncodeToString(h.Sum(nil)), nil - }, - "sha1": func(args ...interface{}) (interface{}, error) { - h := sha1.New() - _, err := h.Write([]byte(types.ToString(args[0]))) - - if err != nil { - return nil, err - } - return hex.EncodeToString(h.Sum(nil)), nil - }, - "mmh3": func(args ...interface{}) (interface{}, error) { - return fmt.Sprintf("%d", int32(murmur3.Sum32WithSeed([]byte(types.ToString(args[0])), 0))), nil - }, - // search - "contains": func(args ...interface{}) (interface{}, error) { - return strings.Contains(types.ToString(args[0]), types.ToString(args[1])), nil - }, - "regex": func(args ...interface{}) (interface{}, error) { - compiled, err := regexp.Compile(types.ToString(args[0])) - if err != nil { - return nil, err - } - return compiled.MatchString(types.ToString(args[1])), nil - }, - // random generators - "rand_char": func(args ...interface{}) (interface{}, error) { - chars := letters + numbers - bad := "" - if len(args) >= 1 { - chars = types.ToString(args[0]) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - chars = trimAll(chars, bad) - return chars[rand.Intn(len(chars))], nil - }, - "rand_base": func(args ...interface{}) (interface{}, error) { - l := 0 - bad := "" - base := letters + numbers - - if len(args) >= 1 { - l = args[0].(int) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - if len(args) >= withBaseRandArgsSize { - base = types.ToString(args[2]) - } - base = trimAll(base, bad) - return randSeq(base, l), nil - }, - "rand_text_alphanumeric": func(args ...interface{}) (interface{}, error) { - l := 0 - bad := "" - chars := letters + numbers - - if len(args) >= 1 { - l = args[0].(int) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - chars = trimAll(chars, bad) - return randSeq(chars, l), nil - }, - "rand_text_alpha": func(args ...interface{}) (interface{}, error) { - l := 0 - bad := "" - chars := letters - - if len(args) >= 1 { - l = args[0].(int) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - chars = trimAll(chars, bad) - return randSeq(chars, l), nil - }, - "rand_text_numeric": func(args ...interface{}) (interface{}, error) { - l := 0 - bad := "" - chars := numbers - - if len(args) >= 1 { - l = args[0].(int) - } - if len(args) >= withCutSetArgsSize { - bad = types.ToString(args[1]) - } - chars = trimAll(chars, bad) - return randSeq(chars, l), nil - }, - "rand_int": func(args ...interface{}) (interface{}, error) { - min := 0 - max := math.MaxInt32 - - if len(args) >= 1 { - min = args[0].(int) - } - if len(args) >= withMaxRandArgsSize { - max = args[1].(int) - } - return rand.Intn(max-min) + min, nil - }, - // Time Functions - "waitfor": func(args ...interface{}) (interface{}, error) { - seconds := args[0].(float64) - time.Sleep(time.Duration(seconds) * time.Second) - return true, nil - }, - // deserialization Functions - "generate_java_gadget": func(args ...interface{}) (interface{}, error) { - gadget := args[0].(string) - cmd := args[1].(string) - - var encoding string - if len(args) > 2 { - encoding = args[2].(string) - } - data := deserialization.GenerateJavaGadget(gadget, cmd, encoding) - return data, nil - }, -} - -// HelperFunctions returns the dsl helper functions -func HelperFunctions() map[string]govaluate.ExpressionFunction { - return functions -} - -// AddHelperFunction allows creation of additiona helper functions to be supported with templates -func AddHelperFunction(key string, value func(args ...interface{}) (interface{}, error)) error { - if _, ok := functions[key]; !ok { - functions[key] = value - return nil - } - return errors.New("duplicate helper function key defined") -} - -func reverseString(s string) string { - runes := []rune(s) - for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { - runes[i], runes[j] = runes[j], runes[i] - } - return string(runes) -} - -func trimAll(s, cutset string) string { - for _, c := range cutset { - s = strings.ReplaceAll(s, string(c), "") - } - return s -} - -func randSeq(base string, n int) string { - b := make([]rune, n) - for i := range b { - b[i] = rune(base[rand.Intn(len(base))]) - } - return string(b) -} diff --git a/v2/pkg/operators/matchers/compile.go b/v2/pkg/operators/matchers/compile.go index dede36652f..912d54b52c 100644 --- a/v2/pkg/operators/matchers/compile.go +++ b/v2/pkg/operators/matchers/compile.go @@ -4,9 +4,6 @@ import ( "encoding/hex" "fmt" "regexp" - - "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" ) // CompileMatchers performs the initial setup operation on a matcher @@ -41,15 +38,6 @@ func (m *Matcher) CompileMatchers() error { m.regexCompiled = append(m.regexCompiled, compiled) } - // Compile the dsl expressions - for _, expr := range m.DSL { - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) - if err != nil { - return fmt.Errorf("could not compile dsl: %s", expr) - } - m.dslCompiled = append(m.dslCompiled, compiled) - } - // Setup the condition type, if any. if m.Condition != "" { m.condition, ok = ConditionTypes[m.Condition] diff --git a/v2/pkg/operators/matchers/match.go b/v2/pkg/operators/matchers/match.go index 2b31386c4a..e13f900769 100644 --- a/v2/pkg/operators/matchers/match.go +++ b/v2/pkg/operators/matchers/match.go @@ -3,6 +3,8 @@ package matchers import ( "encoding/hex" "strings" + + "github.com/projectdiscovery/nebula" ) // MatchStatusCode matches a status code check against a corpus @@ -125,8 +127,8 @@ func (m *Matcher) MatchBinary(corpus string) bool { // MatchDSL matches on a generic map result func (m *Matcher) MatchDSL(data map[string]interface{}) bool { // Iterate over all the expressions accepted as valid - for i, expression := range m.dslCompiled { - result, err := expression.Evaluate(data) + for i, expression := range m.DSL { + result, err := nebula.EvalExp(expression, data) if err != nil { continue } @@ -151,7 +153,7 @@ func (m *Matcher) MatchDSL(data map[string]interface{}) bool { } // If we are at the end of the dsl, return with true - if len(m.dslCompiled)-1 == i { + if len(m.DSL)-1 == i { return true } } diff --git a/v2/pkg/operators/matchers/matchers.go b/v2/pkg/operators/matchers/matchers.go index 2a13b6a1e8..69a0c11d08 100644 --- a/v2/pkg/operators/matchers/matchers.go +++ b/v2/pkg/operators/matchers/matchers.go @@ -2,8 +2,6 @@ package matchers import ( "regexp" - - "github.com/Knetic/govaluate" ) // Matcher is used to match a part in the output from a protocol. @@ -43,7 +41,6 @@ type Matcher struct { condition ConditionType matcherType MatcherType regexCompiled []*regexp.Regexp - dslCompiled []*govaluate.EvaluableExpression } // MatcherType is the type of the matcher specified diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index 7f27b88a38..a09e12df30 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -3,8 +3,7 @@ package expressions import ( "regexp" - "github.com/Knetic/govaluate" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" + "github.com/projectdiscovery/nebula" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" ) @@ -24,14 +23,11 @@ func Evaluate(data string, base map[string]interface{}) (string, error) { for _, match := range templateExpressionRegex.FindAllString(data, -1) { expr := generators.TrimDelimiters(match) - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) - if err != nil { - continue - } - result, err := compiled.Evaluate(base) + result, err := nebula.EvalExp(expr, base) if err != nil { continue } + dynamicValues[expr] = result } // Replacer dynamic values if any in raw request and parse it @@ -45,22 +41,6 @@ func Evaluate(data string, base map[string]interface{}) (string, error) { // The provided keys from finalValues will be used as variable names // for substitution inside the expression. func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) { - final := replacer.Replace(string(data), base) - - dynamicValues := make(map[string]interface{}) - for _, match := range templateExpressionRegex.FindAllString(final, -1) { - expr := generators.TrimDelimiters(match) - - compiled, err := govaluate.NewEvaluableExpressionWithFunctions(expr, dsl.HelperFunctions()) - if err != nil { - continue - } - result, err := compiled.Evaluate(base) - if err != nil { - continue - } - dynamicValues[expr] = result - } - // Replacer dynamic values if any in raw request and parse it - return []byte(replacer.Replace(final, dynamicValues)), nil + strData, err := Evaluate(string(data), base) + return []byte(strData), err } From acb360876c9a903e5266ea7d63e4893377d8530b Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 9 Aug 2021 19:36:11 +0200 Subject: [PATCH 03/15] adding custom helpers --- v2/internal/runner/runner.go | 4 ++++ v2/pkg/operators/common/dsl/dsl.go | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 v2/pkg/operators/common/dsl/dsl.go diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 2d84ff4260..25331f5218 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -16,6 +16,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" @@ -257,6 +258,9 @@ func New(options *types.Options) (*Runner, error) { } else { runner.ratelimiter = ratelimit.NewUnlimited() } + + dsl.AddCustomHelpers() + return runner, nil } diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go new file mode 100644 index 0000000000..f43a0d0684 --- /dev/null +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -0,0 +1,20 @@ +package dsl + +import ( + "github.com/projectdiscovery/nebula" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization" +) + +func AddCustomHelpers() error { + return nebula.AddFunc("generate_java_gadget", func(args ...interface{}) (interface{}, error) { + gadget := args[0].(string) + cmd := args[1].(string) + + var encoding string + if len(args) > 2 { + encoding = args[2].(string) + } + data := deserialization.GenerateJavaGadget(gadget, cmd, encoding) + return data, nil + }) +} From ff7ed67808a9ef75c179f6319b22c3b9e5b938f7 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 10 Aug 2021 09:43:56 +0200 Subject: [PATCH 04/15] adding mergevariables helper + usage in network templates --- v2/pkg/protocols/common/generators/variables.go | 16 ++++++++++++++++ v2/pkg/protocols/http/build_request.go | 10 +--------- v2/pkg/protocols/network/network.go | 6 +++++- 3 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 v2/pkg/protocols/common/generators/variables.go diff --git a/v2/pkg/protocols/common/generators/variables.go b/v2/pkg/protocols/common/generators/variables.go new file mode 100644 index 0000000000..a83a51223c --- /dev/null +++ b/v2/pkg/protocols/common/generators/variables.go @@ -0,0 +1,16 @@ +package generators + +import "github.com/projectdiscovery/nuclei/v2/pkg/types" + +// Init initializes the client pools for the protocols +func MergeVariables(values map[string]interface{}, options *types.Options) map[string]interface{} { + // merge with global vars if any + if !options.Vars.IsEmpty() { + values = MergeMaps(values, options.Vars.AsMap()) + } + + // merge with env vars + if options.EnvironmentVariables { + values = MergeMaps(values, EnvVars()) + } +} diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index a0fa13a1c7..6fb4bd2392 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -62,15 +62,7 @@ func (r *requestGenerator) Make(baseURL string, dynamicValues map[string]interfa parsedString := parsed.String() values["BaseURL"] = parsedString - // merge with vars - if !r.options.Options.Vars.IsEmpty() { - values = generators.MergeMaps(values, r.options.Options.Vars.AsMap()) - } - - // merge with env vars - if r.options.Options.EnvironmentVariables { - values = generators.MergeMaps(values, generators.EnvVars()) - } + values = generators.MergeVariables(values, r.options.Options) // If data contains \n it's a raw request, process it like raw. Else // continue with the template based request flow. diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index 196fcbcde3..3ac8fdbe4f 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -88,12 +88,16 @@ func (r *Request) Compile(options *protocols.ExecuterOptions) error { r.addresses = append(r.addresses, addressKV{ip: address, tls: shouldUseTLS}) } } + + // add global cli vars and environment vars to allow their usage in the payload processing + values := generators.MergeVariables(map[string]interface{}{}, options.Options) + // Pre-compile any input dsl functions before executing the request. for _, input := range r.Inputs { if input.Type != "" { continue } - if compiled, evalErr := expressions.Evaluate(input.Data, map[string]interface{}{}); evalErr == nil { + if compiled, evalErr := expressions.Evaluate(input.Data, values); evalErr == nil { input.Data = compiled } } From 5791f1c246c626ad7587396292bc4f30f9340439 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 10 Aug 2021 17:26:03 +0200 Subject: [PATCH 05/15] Adding global store with shared extractors results todo: implement smart hashing mechanism for easy correlations --- v2/internal/runner/runner.go | 9 +++- v2/pkg/operators/common/dsl/dsl.go | 31 +++++++++++++- v2/pkg/operators/extractors/extractors.go | 2 + v2/pkg/operators/operators.go | 6 +++ .../protocols/common/generators/variables.go | 2 + v2/pkg/protocols/common/runtime/vars.go | 41 +++++++++++++++++++ v2/pkg/protocols/protocols.go | 2 + v2/pkg/templates/workflows.go | 1 + v2/pkg/workflows/execute.go | 5 +++ 9 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 v2/pkg/protocols/common/runtime/vars.go diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index 25331f5218..56cca40b16 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -24,6 +24,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/clusterer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/runtime" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/reporting/exporters/disk" @@ -259,8 +260,6 @@ func New(options *types.Options) (*Runner, error) { runner.ratelimiter = ratelimit.NewUnlimited() } - dsl.AddCustomHelpers() - return runner, nil } @@ -293,6 +292,10 @@ func (r *Runner) RunEnumeration() error { r.options.ExcludeTags = append(r.options.ExcludeTags, ignoreFile.Tags...) r.options.ExcludedTemplates = append(r.options.ExcludedTemplates, ignoreFile.Files...) + s, _ := runtime.New() + + dsl.AddGlobalCustomHelpers(&dsl.Options{Store: s}) + executerOpts := protocols.ExecuterOptions{ Output: r.output, Options: r.options, @@ -303,6 +306,7 @@ func (r *Runner) RunEnumeration() error { Interactsh: r.interactsh, ProjectFile: r.projectFile, Browser: r.browser, + Store: s, } loaderConfig := loader.Config{ Templates: r.options.Templates, @@ -414,6 +418,7 @@ func (r *Runner) RunEnumeration() error { Browser: r.browser, ProjectFile: r.projectFile, Interactsh: r.interactsh, + Store: s, } clusterID := fmt.Sprintf("cluster-%s", xid.New().String()) diff --git a/v2/pkg/operators/common/dsl/dsl.go b/v2/pkg/operators/common/dsl/dsl.go index f43a0d0684..2d6fccc4f3 100644 --- a/v2/pkg/operators/common/dsl/dsl.go +++ b/v2/pkg/operators/common/dsl/dsl.go @@ -3,10 +3,15 @@ package dsl import ( "github.com/projectdiscovery/nebula" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/deserialization" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/runtime" ) -func AddCustomHelpers() error { - return nebula.AddFunc("generate_java_gadget", func(args ...interface{}) (interface{}, error) { +type Options struct { + Store *runtime.Store +} + +func AddGlobalCustomHelpers(options *Options) error { + _ = nebula.AddFunc("generate_java_gadget", func(args ...interface{}) (interface{}, error) { gadget := args[0].(string) cmd := args[1].(string) @@ -17,4 +22,26 @@ func AddCustomHelpers() error { data := deserialization.GenerateJavaGadget(gadget, cmd, encoding) return data, nil }) + + _ = nebula.AddFunc("nuclei_vars_set", func(key string, value interface{}) { + options.Store.Set(key, value) + }) + + _ = nebula.AddFunc("nuclei_vars_get", func(key string) (interface{}, error) { + return options.Store.Get(key), nil + }) + + _ = nebula.AddFunc("nuclei_vars_del", func(key string) { + options.Store.Del(key) + }) + + _ = nebula.AddFunc("nuclei_vars_len", func(args ...interface{}) int { + return options.Store.Len() + }) + + _ = nebula.AddFunc("nuclei_vars_has", func(key string) bool { + return options.Store.Has(key) + }) + + return nil } diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index e3a2f051fa..013df7fde3 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -40,6 +40,8 @@ type Extractor struct { Part string `yaml:"part,omitempty"` // Internal defines if this is used internally Internal bool `yaml:"internal,omitempty"` + // Global defines if this should be available globally + Global bool `yaml:"global,omitempty"` } // ExtractorType is the type of the extractor specified diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 2497fa494a..28109e0b29 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -63,6 +63,8 @@ type Result struct { DynamicValues map[string]interface{} // PayloadValues contains payload values provided by user. (Optional) PayloadValues map[string]interface{} + // GlobalValues contains values to be exported to other templates (Optional) + GlobalValues map[string]interface{} } // Merge merges a result structure into the other. @@ -104,6 +106,7 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac Matches: make(map[string]struct{}), Extracts: make(map[string][]string), DynamicValues: make(map[string]interface{}), + GlobalValues: make(map[string]interface{}), } // Start with the extractors first and evaluate them. @@ -123,6 +126,9 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac } if len(extractorResults) > 0 && !extractor.Internal && extractor.Name != "" { result.Extracts[extractor.Name] = extractorResults + if extractor.Global { + result.GlobalValues[extractor.Name] = extractorResults + } } } diff --git a/v2/pkg/protocols/common/generators/variables.go b/v2/pkg/protocols/common/generators/variables.go index a83a51223c..86ebba6bde 100644 --- a/v2/pkg/protocols/common/generators/variables.go +++ b/v2/pkg/protocols/common/generators/variables.go @@ -13,4 +13,6 @@ func MergeVariables(values map[string]interface{}, options *types.Options) map[s if options.EnvironmentVariables { values = MergeMaps(values, EnvVars()) } + + return values } diff --git a/v2/pkg/protocols/common/runtime/vars.go b/v2/pkg/protocols/common/runtime/vars.go new file mode 100644 index 0000000000..3384afe895 --- /dev/null +++ b/v2/pkg/protocols/common/runtime/vars.go @@ -0,0 +1,41 @@ +package runtime + +type Store struct { + kv map[string]interface{} +} + +func New() (*Store, error) { + return &Store{kv: map[string]interface{}{}}, nil +} + +func (store *Store) Reset() { + for k := range store.kv { + // removes var references + delete(store.kv, k) + } +} + +func (store *Store) Set(key string, value interface{}) { + store.kv[key] = value +} + +func (store *Store) Get(key string) interface{} { + v, ok := store.kv[key] + if ok { + return v + } + return nil +} + +func (store *Store) Has(key string) bool { + _, ok := store.kv[key] + return ok +} + +func (store *Store) Del(key string) { + delete(store.kv, key) +} + +func (store *Store) Len() int { + return len(store.kv) +} diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 9225c523f3..66019a656a 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/projectfile" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/runtime" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v2/pkg/reporting" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -55,6 +56,7 @@ type ExecuterOptions struct { Interactsh *interactsh.Client Operators []*operators.Operators // only used by offlinehttp module + Store *runtime.Store } // Request is an interface implemented any protocol based request generator. diff --git a/v2/pkg/templates/workflows.go b/v2/pkg/templates/workflows.go index 628365d0a2..d26fcadc41 100644 --- a/v2/pkg/templates/workflows.go +++ b/v2/pkg/templates/workflows.go @@ -67,6 +67,7 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr IssuesClient: options.IssuesClient, Interactsh: options.Interactsh, ProjectFile: options.ProjectFile, + Store: options.Store, } template, err := Parse(path, preprocessor, opts) if err != nil { diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index 163cc98243..9899bb02d4 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -46,6 +46,11 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res if len(result.Results) > 0 { firstMatched = true } + + // store the global values regardless of outcome + for k, v := range result.OperatorsResult.GlobalValues { + executer.Options.Store.Set(k, v) + } }) } else { var matched bool From 7e4e77bde9dc557029eab9b8d8867a6e0395ba30 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 10 Aug 2021 18:58:32 +0200 Subject: [PATCH 06/15] Adding support for script callbacks --- v2/pkg/operators/operators.go | 2 ++ v2/pkg/operators/sets/doc.go | 1 + v2/pkg/operators/sets/set.go | 6 ++++++ v2/pkg/protocols/http/request.go | 10 ++++++++++ 4 files changed, 19 insertions(+) create mode 100644 v2/pkg/operators/sets/doc.go create mode 100644 v2/pkg/operators/sets/set.go diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 28109e0b29..2342a2be2f 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -4,6 +4,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/sets" ) // Operators contains the operators that can be applied on protocols @@ -14,6 +15,7 @@ type Operators struct { // Extractors contains the extraction mechanism for the request to identify // and extract parts of the response. Extractors []*extractors.Extractor `yaml:"extractors,omitempty"` + Sets []*sets.Set `yaml:"sets,omitempty"` // MatchersCondition is the condition of the matchers // whether to use AND or OR. Default is OR. MatchersCondition string `yaml:"matchers-condition,omitempty"` diff --git a/v2/pkg/operators/sets/doc.go b/v2/pkg/operators/sets/doc.go new file mode 100644 index 0000000000..5d36b10401 --- /dev/null +++ b/v2/pkg/operators/sets/doc.go @@ -0,0 +1 @@ +package sets diff --git a/v2/pkg/operators/sets/set.go b/v2/pkg/operators/sets/set.go new file mode 100644 index 0000000000..7fbc62209e --- /dev/null +++ b/v2/pkg/operators/sets/set.go @@ -0,0 +1,6 @@ +package sets + +type Set struct { + Name string `yaml:"name,omitempty"` + Code string `yaml:"code"` +} diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 0ece585a98..ad599ea6ad 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nebula" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" @@ -436,6 +437,9 @@ func (r *Request) executeRequest(reqURL string, request *generatedRequest, previ } event.InternalEvent = outputEvent } + + r.Set(finalEvent) + callback(event) return nil } @@ -454,3 +458,9 @@ func (r *Request) setCustomHeaders(req *generatedRequest) { } } } + +func (r *Request) Set(data map[string]interface{}) { + for _, set := range r.Sets { + nebula.Eval(set.Code, data) + } +} From a23ede01bb0350742af4eec56df7c9452cfb2b08 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 10 Aug 2021 21:13:49 +0200 Subject: [PATCH 07/15] Improving expression tokenization --- .../common/expressions/expressions.go | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/v2/pkg/protocols/common/expressions/expressions.go b/v2/pkg/protocols/common/expressions/expressions.go index a09e12df30..38ed0c5b24 100644 --- a/v2/pkg/protocols/common/expressions/expressions.go +++ b/v2/pkg/protocols/common/expressions/expressions.go @@ -1,15 +1,14 @@ package expressions import ( - "regexp" + "strings" "github.com/projectdiscovery/nebula" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/replacer" + "github.com/projectdiscovery/stringsutil" ) -var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}["'\)\}]*`) - // Evaluate checks if the match contains a dynamic variable, for each // found one we will check if it's an expression and can // be compiled, it will be evaluated and the results will be returned. @@ -18,9 +17,8 @@ var templateExpressionRegex = regexp.MustCompile(`(?m)\{\{[^}]+\}\}["'\)\}]*`) // for substitution inside the expression. func Evaluate(data string, base map[string]interface{}) (string, error) { data = replacer.Replace(data, base) - dynamicValues := make(map[string]interface{}) - for _, match := range templateExpressionRegex.FindAllString(data, -1) { + for _, match := range findMatches(data) { expr := generators.TrimDelimiters(match) result, err := nebula.EvalExp(expr, base) @@ -44,3 +42,21 @@ func EvaluateByte(data []byte, base map[string]interface{}) ([]byte, error) { strData, err := Evaluate(string(data), base) return []byte(strData), err } + +func findMatches(data string) []string { + var matches []string + tokens := strings.Split(data, "{{") + for _, token := range tokens { + closingToken := strings.LastIndex(token, "}}") + var match string + if closingToken > 0 { + match = token[:closingToken] + } else { + match = stringsutil.Before(token, "}}") + } + + matches = append(matches, match) + } + + return matches +} From 78bdff2c710f736b9d26027df1667d66055940b5 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 11 Aug 2021 22:48:27 +0200 Subject: [PATCH 08/15] Adding support for python dialect template --- v2/pkg/protocols/code/code.go | 44 +++++++++++ v2/pkg/protocols/code/operators.go | 117 +++++++++++++++++++++++++++++ v2/pkg/protocols/code/request.go | 54 +++++++++++++ v2/pkg/templates/compile.go | 8 +- v2/pkg/templates/templates.go | 3 + 5 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 v2/pkg/protocols/code/code.go create mode 100644 v2/pkg/protocols/code/operators.go create mode 100644 v2/pkg/protocols/code/request.go diff --git a/v2/pkg/protocols/code/code.go b/v2/pkg/protocols/code/code.go new file mode 100644 index 0000000000..48dc074293 --- /dev/null +++ b/v2/pkg/protocols/code/code.go @@ -0,0 +1,44 @@ +package code + +import ( + "github.com/pkg/errors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +// Request contains a code request to execute via nebula +type Request struct { + // Operators for the current request go here. + operators.Operators `yaml:",inline"` + + ID string `yaml:"id"` + + // Name contains the identifier of the request + Code string `yaml:"code"` + + CompiledOperators *operators.Operators + options *protocols.ExecuterOptions +} + +// GetID returns the unique ID of the request if any. +func (r *Request) GetID() string { + return r.ID +} + +// Compile compiles the protocol request for further execution. +func (r *Request) Compile(options *protocols.ExecuterOptions) error { + r.options = options + if len(r.Matchers) > 0 || len(r.Extractors) > 0 { + compiled := &r.Operators + if err := compiled.Compile(); err != nil { + return errors.Wrap(err, "could not compile operators") + } + r.CompiledOperators = compiled + } + return nil +} + +// Requests returns the total number of requests the YAML rule will perform +func (r *Request) Requests() int { + return 1 +} diff --git a/v2/pkg/protocols/code/operators.go b/v2/pkg/protocols/code/operators.go new file mode 100644 index 0000000000..bf134e2ed9 --- /dev/null +++ b/v2/pkg/protocols/code/operators.go @@ -0,0 +1,117 @@ +package code + +import ( + "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Match matches a generic data response again a given matcher +func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) bool { + item, ok := data[matcher.Part] + if !ok { + return false + } + + switch matcher.GetType() { + case matchers.StatusMatcher: + return matcher.Result(matcher.MatchStatusCode(item.(int))) + case matchers.SizeMatcher: + return matcher.Result(matcher.MatchSize(len(types.ToString(item)))) + case matchers.WordsMatcher: + return matcher.Result(matcher.MatchWords(types.ToString(item))) + case matchers.RegexMatcher: + return matcher.Result(matcher.MatchRegex(types.ToString(item))) + case matchers.BinaryMatcher: + return matcher.Result(matcher.MatchBinary(types.ToString(item))) + case matchers.DSLMatcher: + return matcher.Result(matcher.MatchDSL(data)) + } + return false +} + +// Extract performs extracting operation for a extractor on model and returns true or false. +func (r *Request) Extract(data map[string]interface{}, extractor *extractors.Extractor) map[string]struct{} { + item, ok := data[extractor.Part] + if !ok { + return nil + } + itemStr := types.ToString(item) + + switch extractor.GetType() { + case extractors.RegexExtractor: + return extractor.ExtractRegex(itemStr) + case extractors.KValExtractor: + return extractor.ExtractKval(data) + } + return nil +} + +// responseToDSLMap converts a DNS response to a map for use in DSL matching +func (r *Request) responseToDSLMap(res map[string]interface{}, host, matched string) output.InternalEvent { + data := make(map[string]interface{}) + + data = generators.MergeMaps(data, res) + + // In order to return results correctly to nuclei the following fields needs to be populated + data["host"] = "" + data["matched"] = "" + data["request"] = "" + data["response"] = "" + + data["template-id"] = r.options.TemplateID + data["template-info"] = r.options.TemplateInfo + data["template-path"] = r.options.TemplatePath + + return data +} + +// MakeResultEvent creates a result event from internal wrapped event +func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { + if len(wrapped.OperatorsResult.DynamicValues) > 0 { + return nil + } + results := make([]*output.ResultEvent, 0, len(wrapped.OperatorsResult.Matches)+1) + + // If we have multiple matchers with names, write each of them separately. + if len(wrapped.OperatorsResult.Matches) > 0 { + for k := range wrapped.OperatorsResult.Matches { + data := r.makeResultEventItem(wrapped) + data.MatcherName = k + results = append(results, data) + } + } else if len(wrapped.OperatorsResult.Extracts) > 0 { + for k, v := range wrapped.OperatorsResult.Extracts { + data := r.makeResultEventItem(wrapped) + data.ExtractedResults = v + data.ExtractorName = k + results = append(results, data) + } + } else { + data := r.makeResultEventItem(wrapped) + results = append(results, data) + } + return results +} + +func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { + data := &output.ResultEvent{ + TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), + TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), + Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), + Type: "code", + Host: types.ToString(wrapped.InternalEvent["host"]), + Matched: types.ToString(wrapped.InternalEvent["matched"]), + ExtractedResults: wrapped.OperatorsResult.OutputExtracts, + Timestamp: time.Now(), + } + if r.options.Options.JSONRequests { + data.Request = types.ToString(wrapped.InternalEvent["request"]) + data.Response = types.ToString(wrapped.InternalEvent["response"]) + } + return data +} diff --git a/v2/pkg/protocols/code/request.go b/v2/pkg/protocols/code/request.go new file mode 100644 index 0000000000..1b01cc76fb --- /dev/null +++ b/v2/pkg/protocols/code/request.go @@ -0,0 +1,54 @@ +package code + +import ( + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nebula" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" +) + +var _ protocols.Request = &Request{} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (r *Request) ExecuteWithResults(input string, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { + metadata["input"] = input + + if r.options.Options.Debug || r.options.Options.DebugRequests { + gologger.Info().Str("input", input).Msgf("[%s] Code request for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%s", r.Code) + } + + res, err := nebula.Eval(r.Code, metadata) + if err != nil { + r.options.Output.Request(r.options.TemplateID, input, "code", err) + r.options.Progress.IncrementFailedRequestsBy(1) + } + if res == nil { + return errors.Wrap(err, "could not execute code request") + } + r.options.Progress.IncrementRequests() + + r.options.Output.Request(r.options.TemplateID, input, "code", err) + gologger.Verbose().Msgf("[%s] Executed code request for %s", r.options.TemplateID, input) + + if r.options.Options.Debug || r.options.Options.DebugResponse { + gologger.Debug().Msgf("[%s] Dumped code response for %s", r.options.TemplateID, input) + gologger.Print().Msgf("%+v", res) + } + outputEvent := r.responseToDSLMap(metadata, input, input) + for k, v := range previous { + outputEvent[k] = v + } + + event := &output.InternalWrappedEvent{InternalEvent: outputEvent} + if r.CompiledOperators != nil { + result, ok := r.CompiledOperators.Execute(outputEvent, r.Match, r.Extract) + if ok && result != nil { + event.OperatorsResult = result + event.Results = r.MakeResultEvent(event) + } + } + callback(event) + return nil +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 7ec032c8d1..bf301b7e8e 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -55,7 +55,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute options.TemplatePath = filePath // If no requests, and it is also not a workflow, return error. - if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows) == 0 { + if len(template.RequestsDNS)+len(template.RequestsHTTP)+len(template.RequestsFile)+len(template.RequestsNetwork)+len(template.RequestsHeadless)+len(template.Workflows)+len(template.RequestCode) == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) } @@ -122,6 +122,12 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Execute } template.Executer = executer.NewExecuter(requests, &options) } + if len(template.RequestCode) > 0 && !options.Options.OfflineHTTP { + for _, req := range template.RequestCode { + requests = append(requests, req) + } + template.Executer = executer.NewExecuter(requests, &options) + } if template.Executer != nil { err := template.Executer.Compile() if err != nil { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 12c135e4f1..e3ed5cb6bf 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -2,6 +2,7 @@ package templates import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/code" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/file" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" @@ -26,6 +27,8 @@ type Template struct { RequestsNetwork []*network.Request `yaml:"network,omitempty" json:"network"` // RequestsHeadless contains the headless request to make in the template. RequestsHeadless []*headless.Request `yaml:"headless,omitempty" json:"headless"` + // RequestCode contains the nebula script to be executed + RequestCode []*code.Request `yaml:"code,omitempty" json:"code"` // Workflows is a yaml based workflow declaration code. workflows.Workflow `yaml:",inline,omitempty"` From f2225b25d1f711ccdd19a44e97bf3fdefec0797c Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 13 Aug 2021 00:48:23 +0200 Subject: [PATCH 09/15] updating go modules --- v2/go.mod | 3 +-- v2/go.sum | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 0cfeeb21ee..1d0c288538 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,7 +21,6 @@ require ( github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.11 - github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible @@ -56,7 +55,7 @@ require ( go.uber.org/atomic v1.7.0 go.uber.org/multierr v1.6.0 go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20210614182718-04defd469f4e + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index 940bbf2b90..8da72e8975 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -240,8 +240,6 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b h1:XUr8tvMEILhphQPp3TFcIudb5KTOzFeD0pJyDn5+5QI= github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b/go.mod h1:a5Mn24iYVJRUQSkFupGByqykzD+k+wFI8J91zGHuPf8= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= @@ -304,7 +302,6 @@ github.com/mewpkg/hashutil v1.0.1/go.mod h1:DSL30IKicBzaNFwGvhg9BG8YkZj0Nb+d6TDP github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -337,8 +334,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= -github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd h1:kPBRsSG8ZwNHLOBzDM/vfGi/Oaj/dzdbTeEkv+6UhEw= -github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd/go.mod h1:C8Cevz76yJLH0RpmIa+wWTVnZQyIL6CyhTnJJl6hHas= +github.com/projectdiscovery/executil v0.0.0-20210804143524-823f37935565 h1:oUSOmpbnQqHiVf8VRPhaEZyWBXYKSnb3EVL50JczMB8= +github.com/projectdiscovery/executil v0.0.0-20210804143524-823f37935565/go.mod h1:124/FKhihb7SX7YF6b7JM3MWgFa4F+nwUxSYZR1UZVQ= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= @@ -459,8 +456,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210812204632-0ba0e8f03122 h1:AOT7vJYHE32m61R8d1WlcqhOO1AocesDsKpcMq+UOaA= +golang.org/x/crypto v0.0.0-20210812204632-0ba0e8f03122/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -523,11 +521,9 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210502030024-e5908800b52b/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -584,9 +580,10 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -755,8 +752,9 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -lukechampine.com/blake3 v1.1.4 h1:cVl4fhMGfCaCFrs4sBb8R/iEMeL0g4CfQp/sAHBDxOI= lukechampine.com/blake3 v1.1.4/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= +lukechampine.com/blake3 v1.1.5 h1:hsACfxWvLdGmjYbWGrumQIphOvO+ZruZehWtgd2fxoM= +lukechampine.com/blake3 v1.1.5/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 0607c52f0de55c59d669c439c6cd80ef59d4980f Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 13 Aug 2021 00:52:31 +0200 Subject: [PATCH 10/15] again go modules --- v2/go.mod | 8 +++++--- v2/go.sum | 13 +++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 1d0c288538..f4aa59b4ed 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -2,8 +2,6 @@ module github.com/projectdiscovery/nuclei/v2 go 1.15 -replace github.com/projectdiscovery/nebula => /Users/marcornvh/go/src/github.com/projectdiscovery/nebula - require ( github.com/andygrunwald/go-jira v1.13.0 github.com/antchfx/htmlquery v1.2.3 @@ -30,12 +28,13 @@ require ( github.com/owenrumney/go-sarif v1.0.4 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.8 + github.com/projectdiscovery/executil v0.0.0-20210804143524-823f37935565 // indirect github.com/projectdiscovery/fastdialer v0.0.8 github.com/projectdiscovery/goflags v0.0.7 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.1 github.com/projectdiscovery/interactsh v0.0.4 - github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49 + github.com/projectdiscovery/nebula v0.0.0-20210812224441-37b4e61899e8 github.com/projectdiscovery/rawhttp v0.0.7 github.com/projectdiscovery/retryabledns v1.0.10 github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727 @@ -55,9 +54,12 @@ require ( go.uber.org/atomic v1.7.0 go.uber.org/multierr v1.6.0 go.uber.org/ratelimit v0.2.0 + golang.org/x/crypto v0.0.0-20210812204632-0ba0e8f03122 // indirect golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 + golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v2 v2.4.0 + lukechampine.com/blake3 v1.1.5 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 8da72e8975..2f12e66c80 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -302,6 +302,7 @@ github.com/mewpkg/hashutil v1.0.1/go.mod h1:DSL30IKicBzaNFwGvhg9BG8YkZj0Nb+d6TDP github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.38/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -334,10 +335,13 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= +github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd h1:kPBRsSG8ZwNHLOBzDM/vfGi/Oaj/dzdbTeEkv+6UhEw= +github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd/go.mod h1:C8Cevz76yJLH0RpmIa+wWTVnZQyIL6CyhTnJJl6hHas= github.com/projectdiscovery/executil v0.0.0-20210804143524-823f37935565 h1:oUSOmpbnQqHiVf8VRPhaEZyWBXYKSnb3EVL50JczMB8= github.com/projectdiscovery/executil v0.0.0-20210804143524-823f37935565/go.mod h1:124/FKhihb7SX7YF6b7JM3MWgFa4F+nwUxSYZR1UZVQ= github.com/projectdiscovery/fastdialer v0.0.8 h1:mEMc8bfXV5hc1PUEkJiUnR5imYQe6+839Zezd5jLkc0= github.com/projectdiscovery/fastdialer v0.0.8/go.mod h1:AuaV0dzrNeBLHqjNnzpFSnTXnHGIZAlGQE+WUMmSIW4= +github.com/projectdiscovery/fileutil v0.0.0-20210601061022-8ef4fc6fbfb6/go.mod h1:+KrA1FQf8IYdgf/ifBw+7GXW+zYuppvqJKb+lmloDVw= github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= @@ -348,6 +352,10 @@ github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvz github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= github.com/projectdiscovery/mapsutil v0.0.1 h1:hXIQUPvU+5HtPzzyaLvsPg4R/39N77CCAL0ezYhFdgw= github.com/projectdiscovery/mapsutil v0.0.1/go.mod h1:Q49P7sQzz9m+dWgdx/ryNwOfsClt8ZmHhF4WdXTheyE= +github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49 h1:ktRs9zNmuGovhewcQ1qadnwMEclvPGvUxVSHwJsNjMY= +github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49/go.mod h1:yRk2l4QV4G+biDC4jW6Vuiv9OVMq15V/U072fxH4Gnc= +github.com/projectdiscovery/nebula v0.0.0-20210812224441-37b4e61899e8 h1:VcDFFNUh8qb5/OWnOqeV53gv7qcThbO5wu95mXIGyF8= +github.com/projectdiscovery/nebula v0.0.0-20210812224441-37b4e61899e8/go.mod h1:snzwYWdwuDmCBzPtSPC1d5noN1tXljedoiAJlVorKoc= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= github.com/projectdiscovery/rawhttp v0.0.7/go.mod h1:PQERZAhAv7yxI/hR6hdDPgK1WTU56l204BweXrBec+0= github.com/projectdiscovery/retryabledns v1.0.7/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ= @@ -456,6 +464,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210812204632-0ba0e8f03122 h1:AOT7vJYHE32m61R8d1WlcqhOO1AocesDsKpcMq+UOaA= golang.org/x/crypto v0.0.0-20210812204632-0ba0e8f03122/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -521,6 +530,7 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210502030024-e5908800b52b/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -580,6 +590,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= @@ -752,6 +764,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/blake3 v1.1.4 h1:cVl4fhMGfCaCFrs4sBb8R/iEMeL0g4CfQp/sAHBDxOI= lukechampine.com/blake3 v1.1.4/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= lukechampine.com/blake3 v1.1.5 h1:hsACfxWvLdGmjYbWGrumQIphOvO+ZruZehWtgd2fxoM= lukechampine.com/blake3 v1.1.5/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= From 2205cbb38941b31df3ca8fc2e80777e4777fd5e9 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 14 Aug 2021 03:53:05 +0200 Subject: [PATCH 11/15] applying changes from dev --- v2/go.sum | 6 ------ v2/pkg/protocols/common/generators/variables.go | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/v2/go.sum b/v2/go.sum index 2f12e66c80..91d7f032a8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -335,7 +335,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= -github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd h1:kPBRsSG8ZwNHLOBzDM/vfGi/Oaj/dzdbTeEkv+6UhEw= github.com/projectdiscovery/executil v0.0.0-20210414225944-2ad029b6a1fd/go.mod h1:C8Cevz76yJLH0RpmIa+wWTVnZQyIL6CyhTnJJl6hHas= github.com/projectdiscovery/executil v0.0.0-20210804143524-823f37935565 h1:oUSOmpbnQqHiVf8VRPhaEZyWBXYKSnb3EVL50JczMB8= github.com/projectdiscovery/executil v0.0.0-20210804143524-823f37935565/go.mod h1:124/FKhihb7SX7YF6b7JM3MWgFa4F+nwUxSYZR1UZVQ= @@ -352,8 +351,6 @@ github.com/projectdiscovery/interactsh v0.0.4 h1:3BtCZrrTovGYiqdFktXJ4NxKAQFvUvz github.com/projectdiscovery/interactsh v0.0.4/go.mod h1:PtJrddeBW1/LeOVgTvvnjUl3Hu/17jTkoIi8rXeEODE= github.com/projectdiscovery/mapsutil v0.0.1 h1:hXIQUPvU+5HtPzzyaLvsPg4R/39N77CCAL0ezYhFdgw= github.com/projectdiscovery/mapsutil v0.0.1/go.mod h1:Q49P7sQzz9m+dWgdx/ryNwOfsClt8ZmHhF4WdXTheyE= -github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49 h1:ktRs9zNmuGovhewcQ1qadnwMEclvPGvUxVSHwJsNjMY= -github.com/projectdiscovery/nebula v0.0.0-20210802111745-ce1775478b49/go.mod h1:yRk2l4QV4G+biDC4jW6Vuiv9OVMq15V/U072fxH4Gnc= github.com/projectdiscovery/nebula v0.0.0-20210812224441-37b4e61899e8 h1:VcDFFNUh8qb5/OWnOqeV53gv7qcThbO5wu95mXIGyF8= github.com/projectdiscovery/nebula v0.0.0-20210812224441-37b4e61899e8/go.mod h1:snzwYWdwuDmCBzPtSPC1d5noN1tXljedoiAJlVorKoc= github.com/projectdiscovery/rawhttp v0.0.7 h1:5m4peVgjbl7gqDcRYMTVEuX+Xs/nh76ohTkkvufucLg= @@ -464,7 +461,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210812204632-0ba0e8f03122 h1:AOT7vJYHE32m61R8d1WlcqhOO1AocesDsKpcMq+UOaA= golang.org/x/crypto v0.0.0-20210812204632-0ba0e8f03122/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -591,7 +587,6 @@ golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b h1:qh4f65QIVFjq9eBURLEYWqaEXmOyqdUyiBSgaXWccWk= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= @@ -764,7 +759,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -lukechampine.com/blake3 v1.1.4 h1:cVl4fhMGfCaCFrs4sBb8R/iEMeL0g4CfQp/sAHBDxOI= lukechampine.com/blake3 v1.1.4/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= lukechampine.com/blake3 v1.1.5 h1:hsACfxWvLdGmjYbWGrumQIphOvO+ZruZehWtgd2fxoM= lukechampine.com/blake3 v1.1.5/go.mod h1:hE8RpzdO8ttZ7446CXEwDP1eu2V4z7stv0Urj1El20g= diff --git a/v2/pkg/protocols/common/generators/variables.go b/v2/pkg/protocols/common/generators/variables.go index 86ebba6bde..e6b0c72cec 100644 --- a/v2/pkg/protocols/common/generators/variables.go +++ b/v2/pkg/protocols/common/generators/variables.go @@ -2,7 +2,7 @@ package generators import "github.com/projectdiscovery/nuclei/v2/pkg/types" -// Init initializes the client pools for the protocols +// Merge the variables to the map of values func MergeVariables(values map[string]interface{}, options *types.Options) map[string]interface{} { // merge with global vars if any if !options.Vars.IsEmpty() { @@ -11,7 +11,7 @@ func MergeVariables(values map[string]interface{}, options *types.Options) map[s // merge with env vars if options.EnvironmentVariables { - values = MergeMaps(values, EnvVars()) + values = MergeMaps(EnvVars(), values) } return values From 1ed1f850b249a148d420b69af90842e1ce487f44 Mon Sep 17 00:00:00 2001 From: mzack Date: Tue, 17 Aug 2021 14:51:06 +0200 Subject: [PATCH 12/15] merging more changes from dev --- v2/go.mod | 1 + v2/go.sum | 2 ++ v2/pkg/protocols/code/operators.go | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/v2/go.mod b/v2/go.mod index f4aa59b4ed..4165fed5d3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -19,6 +19,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/itchyny/gojq v0.12.4 github.com/json-iterator/go v1.1.11 + github.com/julienschmidt/httprouter v1.3.0 github.com/karlseguin/ccache v2.0.3+incompatible github.com/karrick/godirwalk v1.16.1 github.com/logrusorgru/aurora v2.0.3+incompatible diff --git a/v2/go.sum b/v2/go.sum index 91d7f032a8..52e2e7a17c 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -240,6 +240,8 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b h1:XUr8tvMEILhphQPp3TFcIudb5KTOzFeD0pJyDn5+5QI= github.com/jxskiss/base62 v0.0.0-20191017122030-4f11678b909b/go.mod h1:a5Mn24iYVJRUQSkFupGByqykzD+k+wFI8J91zGHuPf8= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= diff --git a/v2/pkg/protocols/code/operators.go b/v2/pkg/protocols/code/operators.go index bf134e2ed9..28b77687d9 100644 --- a/v2/pkg/protocols/code/operators.go +++ b/v2/pkg/protocols/code/operators.go @@ -3,6 +3,7 @@ package code import ( "time" + "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -102,7 +103,7 @@ func (r *Request) makeResultEventItem(wrapped *output.InternalWrappedEvent) *out data := &output.ResultEvent{ TemplateID: types.ToString(wrapped.InternalEvent["template-id"]), TemplatePath: types.ToString(wrapped.InternalEvent["template-path"]), - Info: wrapped.InternalEvent["template-info"].(map[string]interface{}), + Info: wrapped.InternalEvent["template-info"].(model.Info), Type: "code", Host: types.ToString(wrapped.InternalEvent["host"]), Matched: types.ToString(wrapped.InternalEvent["matched"]), From 676e1d7354079ac334e3717afb2d54679cee5374 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 13 Sep 2021 18:54:43 +0200 Subject: [PATCH 13/15] making extractors available to all templates within a workflow as dynamic values --- v2/internal/runner/processor.go | 2 +- v2/pkg/operators/extractors/extractors.go | 2 ++ v2/pkg/operators/operators.go | 16 +++++++++++----- v2/pkg/protocols/common/clusterer/executer.go | 6 ++++-- v2/pkg/protocols/common/executer/executer.go | 7 +++++-- v2/pkg/protocols/protocols.go | 4 ++-- v2/pkg/workflows/execute.go | 18 ++++++++++-------- v2/pkg/workflows/execute_test.go | 4 ++-- 8 files changed, 37 insertions(+), 22 deletions(-) diff --git a/v2/internal/runner/processor.go b/v2/internal/runner/processor.go index 6006823ec1..f03329bad5 100644 --- a/v2/internal/runner/processor.go +++ b/v2/internal/runner/processor.go @@ -22,7 +22,7 @@ func (r *Runner) processTemplateWithList(template *templates.Template) bool { go func(URL string) { defer wg.Done() - match, err := template.Executer.Execute(URL) + match, err := template.Executer.Execute(URL, nil) if err != nil { gologger.Warning().Msgf("[%s] Could not execute step: %s\n", r.colorizer.BrightBlue(template.ID), err) } diff --git a/v2/pkg/operators/extractors/extractors.go b/v2/pkg/operators/extractors/extractors.go index 5ae806469c..049a52f7b7 100644 --- a/v2/pkg/operators/extractors/extractors.go +++ b/v2/pkg/operators/extractors/extractors.go @@ -105,6 +105,8 @@ type Extractor struct { // Internal, when set to true will allow using the value extracted // in the next request for some protocols (like HTTP). Internal bool `yaml:"internal,omitempty" jsonschema:"title=mark extracted value for internal variable use,description=Internal when set to true will allow using the value extracted in the next request for some protocols"` + // Parametrizable defines if this should be available within a workflow + Parametrizable bool `yaml:"parametrizable,omitempty"` // Global defines if this should be available globally Global bool `yaml:"global,omitempty"` } diff --git a/v2/pkg/operators/operators.go b/v2/pkg/operators/operators.go index 3220961c78..59e359a9dd 100644 --- a/v2/pkg/operators/operators.go +++ b/v2/pkg/operators/operators.go @@ -51,6 +51,7 @@ func (r *Operators) Compile() error { return errors.Wrap(err, "could not compile extractor") } } + return nil } @@ -77,6 +78,8 @@ type Result struct { PayloadValues map[string]interface{} // GlobalValues contains values to be exported to other templates (Optional) GlobalValues map[string]interface{} + // ParametrizedValues contains values to be exported to other workflow templates (Optional) + ParametrizedValues map[string]interface{} } // Merge merges a result structure into the other. @@ -115,10 +118,11 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac var matches bool result := &Result{ - Matches: make(map[string]struct{}), - Extracts: make(map[string][]string), - DynamicValues: make(map[string]interface{}), - GlobalValues: make(map[string]interface{}), + Matches: make(map[string]struct{}), + Extracts: make(map[string][]string), + DynamicValues: make(map[string]interface{}), + GlobalValues: make(map[string]interface{}), + ParametrizedValues: make(map[string]interface{}), } // Start with the extractors first and evaluate them. @@ -141,6 +145,9 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac if extractor.Global { result.GlobalValues[extractor.Name] = extractorResults } + if extractor.Parametrizable { + result.ParametrizedValues[extractor.Name] = extractorResults + } } } @@ -184,7 +191,6 @@ func (r *Operators) Execute(data map[string]interface{}, match MatchFunc, extrac // ExecuteInternalExtractors executes internal dynamic extractors func (r *Operators) ExecuteInternalExtractors(data map[string]interface{}, extract ExtractFunc) map[string]interface{} { dynamicValues := make(map[string]interface{}) - // Start with the extractors first and evaluate them. for _, extractor := range r.Extractors { if !extractor.Internal { diff --git a/v2/pkg/protocols/common/clusterer/executer.go b/v2/pkg/protocols/common/clusterer/executer.go index ac523c8d61..50bba3d0d8 100644 --- a/v2/pkg/protocols/common/clusterer/executer.go +++ b/v2/pkg/protocols/common/clusterer/executer.go @@ -2,6 +2,7 @@ package clusterer import ( "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/mapsutil" "github.com/projectdiscovery/nuclei/v2/pkg/model" "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" @@ -60,11 +61,12 @@ func (e *Executer) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input string) (bool, error) { +func (e *Executer) Execute(input string, params map[string]interface{}) (bool, error) { var results bool previous := make(map[string]interface{}) dynamicValues := make(map[string]interface{}) + dynamicValues = mapsutil.MergeMaps(dynamicValues, params) err := e.requests.ExecuteWithResults(input, dynamicValues, previous, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { result, matched := operator.operator.Execute(event.InternalEvent, e.requests.Match, e.requests.Extract) @@ -94,7 +96,7 @@ func (e *Executer) Execute(input string) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { +func (e *Executer) ExecuteWithResults(input string, params map[string]interface{}, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) err := e.requests.ExecuteWithResults(input, dynamicValues, nil, func(event *output.InternalWrappedEvent) { for _, operator := range e.operators { diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 65ea5eb398..86311b42f9 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/mapsutil" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" ) @@ -42,10 +43,11 @@ func (e *Executer) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input string) (bool, error) { +func (e *Executer) Execute(input string, params map[string]interface{}) (bool, error) { var results bool dynamicValues := make(map[string]interface{}) + dynamicValues = mapsutil.MergeMaps(dynamicValues, params) previous := make(map[string]interface{}) for _, req := range e.requests { req := req @@ -89,8 +91,9 @@ func (e *Executer) Execute(input string) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { +func (e *Executer) ExecuteWithResults(input string, params map[string]interface{}, callback protocols.OutputEventCallback) error { dynamicValues := make(map[string]interface{}) + dynamicValues = mapsutil.MergeMaps(dynamicValues, params) previous := make(map[string]interface{}) for _, req := range e.requests { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 64ee0bed28..ba9b11754f 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -25,9 +25,9 @@ type Executer interface { // Requests returns the total number of requests the rule will perform Requests() int // Execute executes the protocol group and returns true or false if results were found. - Execute(input string) (bool, error) + Execute(input string, params map[string]interface{}) (bool, error) // ExecuteWithResults executes the protocol requests and returns results instead of writing them. - ExecuteWithResults(input string, callback OutputEventCallback) error + ExecuteWithResults(input string, params map[string]interface{}, callback OutputEventCallback) error } // ExecuterOptions contains the configuration options for executer clients diff --git a/v2/pkg/workflows/execute.go b/v2/pkg/workflows/execute.go index ab44c44374..ef2417fb15 100644 --- a/v2/pkg/workflows/execute.go +++ b/v2/pkg/workflows/execute.go @@ -15,7 +15,7 @@ func (w *Workflow) RunWorkflow(input string) bool { for _, template := range w.Workflows { swg.Add() func(template *WorkflowTemplate) { - err := w.runWorkflowStep(template, input, results, &swg) + err := w.runWorkflowStep(template, input, nil, results, &swg) if err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } @@ -28,7 +28,7 @@ func (w *Workflow) RunWorkflow(input string) bool { // runWorkflowStep runs a workflow step for the workflow. It executes the workflow // in a recursive manner running all subtemplates and matchers. -func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup) error { +func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, params map[string]interface{}, results *atomic.Bool, swg *sizedwaitgroup.SizedWaitGroup) error { var firstMatched bool var err error var mainErr error @@ -39,7 +39,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res // Don't print results with subtemplates, only print results on template. if len(template.Subtemplates) > 0 { - err = executer.Executer.ExecuteWithResults(input, func(result *output.InternalWrappedEvent) { + err = executer.Executer.ExecuteWithResults(input, params, func(result *output.InternalWrappedEvent) { if result.OperatorsResult == nil { return } @@ -51,10 +51,13 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res for k, v := range result.OperatorsResult.GlobalValues { executer.Options.Store.Set(k, v) } + if len(result.OperatorsResult.ParametrizedValues) > 0 { + params = result.OperatorsResult.ParametrizedValues + } }) } else { var matched bool - matched, err = executer.Executer.Execute(input) + matched, err = executer.Executer.Execute(input, params) if matched { firstMatched = true } @@ -80,8 +83,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res if len(template.Matchers) > 0 { for _, executer := range template.Executers { executer.Options.Progress.AddToTotal(int64(executer.Executer.Requests())) - - err := executer.Executer.ExecuteWithResults(input, func(event *output.InternalWrappedEvent) { + err := executer.Executer.ExecuteWithResults(input, params, func(event *output.InternalWrappedEvent) { if event.OperatorsResult == nil { return } @@ -97,7 +99,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res swg.Add() go func(subtemplate *WorkflowTemplate) { - if err := w.runWorkflowStep(subtemplate, input, results, swg); err != nil { + if err := w.runWorkflowStep(subtemplate, input, params, results, swg); err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", subtemplate.Template, err) } swg.Done() @@ -121,7 +123,7 @@ func (w *Workflow) runWorkflowStep(template *WorkflowTemplate, input string, res swg.Add() go func(template *WorkflowTemplate) { - err := w.runWorkflowStep(template, input, results, swg) + err := w.runWorkflowStep(template, input, params, results, swg) if err != nil { gologger.Warning().Msgf("[%s] Could not execute workflow step: %s\n", template.Template, err) } diff --git a/v2/pkg/workflows/execute_test.go b/v2/pkg/workflows/execute_test.go index 26f7c2b994..387dc9b5e2 100644 --- a/v2/pkg/workflows/execute_test.go +++ b/v2/pkg/workflows/execute_test.go @@ -169,7 +169,7 @@ func (m *mockExecuter) Requests() int { } // Execute executes the protocol group and returns true or false if results were found. -func (m *mockExecuter) Execute(input string) (bool, error) { +func (m *mockExecuter) Execute(input string, params map[string]interface{}) (bool, error) { if m.executeHook != nil { m.executeHook(input) } @@ -177,7 +177,7 @@ func (m *mockExecuter) Execute(input string) (bool, error) { } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (m *mockExecuter) ExecuteWithResults(input string, callback protocols.OutputEventCallback) error { +func (m *mockExecuter) ExecuteWithResults(input string, params map[string]interface{}, callback protocols.OutputEventCallback) error { if m.executeHook != nil { m.executeHook(input) } From bd87ecfec980a1fec928365c8d5b644b8e70d6d9 Mon Sep 17 00:00:00 2001 From: mzack Date: Sat, 18 Sep 2021 21:50:27 +0200 Subject: [PATCH 14/15] misc (tests need fix) --- v2/internal/runner/runner.go | 2 +- v2/pkg/operators/common/dsl/dsl_test.go | 7 +++---- v2/pkg/protocols/http/request.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/v2/internal/runner/runner.go b/v2/internal/runner/runner.go index cb7ab77468..1eef64f481 100644 --- a/v2/internal/runner/runner.go +++ b/v2/internal/runner/runner.go @@ -315,7 +315,7 @@ func (r *Runner) RunEnumeration() error { s, _ := runtime.New() - dsl.AddGlobalCustomHelpers(&dsl.Options{Store: s}) + _ = dsl.AddGlobalCustomHelpers(&dsl.Options{Store: s}) var cache *hosterrorscache.Cache if r.options.MaxHostError > 0 { diff --git a/v2/pkg/operators/common/dsl/dsl_test.go b/v2/pkg/operators/common/dsl/dsl_test.go index bf2c5bfefe..b36cf96331 100644 --- a/v2/pkg/operators/common/dsl/dsl_test.go +++ b/v2/pkg/operators/common/dsl/dsl_test.go @@ -3,17 +3,16 @@ package dsl import ( "testing" + "github.com/projectdiscovery/nebula" "github.com/stretchr/testify/require" ) func TestDSLURLEncodeDecode(t *testing.T) { - functions := HelperFunctions() - - encoded, err := functions["url_encode"]("&test\"") + encoded, err := nebula.EvalExp("url_encode('&test\"')", nil) require.Nil(t, err, "could not url encode") require.Equal(t, "%26test%22", encoded, "could not get url encoded data") - decoded, err := functions["url_decode"]("%26test%22") + decoded, err := nebula.EvalExp("url_decode('%26test%22')", nil) require.Nil(t, err, "could not url encode") require.Equal(t, "&test\"", decoded, "could not get url decoded data") } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index a3f3304117..54cde8234b 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -501,6 +501,6 @@ func (r *Request) setCustomHeaders(req *generatedRequest) { func (r *Request) Set(data map[string]interface{}) { for _, set := range r.Sets { - nebula.Eval(set.Code, data) + _, _ = nebula.Eval(set.Code, data) } } From 3adbb6035d3295ca960564776ca18bc37fc121d5 Mon Sep 17 00:00:00 2001 From: mzack Date: Fri, 8 Oct 2021 19:31:28 +0200 Subject: [PATCH 15/15] merging dev --- v2/pkg/protocols/code/operators.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/protocols/code/operators.go b/v2/pkg/protocols/code/operators.go index 28b77687d9..6544b7f047 100644 --- a/v2/pkg/protocols/code/operators.go +++ b/v2/pkg/protocols/code/operators.go @@ -24,7 +24,7 @@ func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) case matchers.SizeMatcher: return matcher.Result(matcher.MatchSize(len(types.ToString(item)))) case matchers.WordsMatcher: - return matcher.Result(matcher.MatchWords(types.ToString(item))) + return matcher.Result(matcher.MatchWords(types.ToString(item), nil)) case matchers.RegexMatcher: return matcher.Result(matcher.MatchRegex(types.ToString(item))) case matchers.BinaryMatcher: