From 94656f835448ebafcb8f51306d736a5aa45d2bcd Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 1 Aug 2023 22:33:16 +0530 Subject: [PATCH 01/38] add flow logic --- v2/go.mod | 12 +- v2/go.sum | 27 ++-- v2/pkg/protocols/common/executer/executer.go | 151 ++++++++++-------- .../common/executer/flow_executor.go | 149 +++++++++++++++++ v2/pkg/protocols/protocols.go | 4 +- v2/pkg/templates/compile.go | 6 + v2/pkg/templates/templates.go | 12 ++ 7 files changed, 277 insertions(+), 84 deletions(-) create mode 100644 v2/pkg/protocols/common/executer/flow_executor.go diff --git a/v2/go.mod b/v2/go.mod index 28ddc1ded3..cc2fd428ac 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -39,16 +39,16 @@ require ( github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.11.0 + golang.org/x/net v0.12.0 golang.org/x/oauth2 v0.9.0 - golang.org/x/text v0.10.0 + golang.org/x/text v0.11.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/DataDog/gostackparse v0.6.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 @@ -87,7 +87,7 @@ require ( require ( aead.dev/minisign v0.2.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/Mzack9999/gostruct v0.0.0-20230415193108-30b70932da81 // indirect @@ -216,10 +216,10 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect - golang.org/x/crypto v0.10.0 + golang.org/x/crypto v0.11.0 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 golang.org/x/mod v0.11.0 // indirect - golang.org/x/sys v0.9.0 // indirect + golang.org/x/sys v0.10.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index 5aa2d42bcf..31228c2af1 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -3,14 +3,15 @@ aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a h1:3i+FJ7IpSZHL+VAjtpQeZCRhrpP0odl5XfoLBY4fxJ8= git.mills.io/prologic/smtpd v0.0.0-20210710122116-a525b76c287a/go.mod h1:C7hXLmFmPYPjIDGfQl1clsmQ5TMEQfmzWTrJk475bUs= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/DataDog/gostackparse v0.6.0 h1:egCGQviIabPwsyoWpGvIBGrEnNWez35aEO7OJ1vBI4o= @@ -629,8 +630,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -663,8 +664,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= @@ -715,8 +716,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -725,7 +726,7 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -735,8 +736,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index c25733a4ad..248dac81c5 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -13,6 +13,8 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" + "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + mapsutil "github.com/projectdiscovery/utils/maps" ) // Executer executes a group of requests for a protocol @@ -61,8 +63,47 @@ func (e *Executer) Requests() int { // Execute executes the protocol group and returns true or false if results were found. func (e *Executer) Execute(input *contextargs.Context) (bool, error) { - results := &atomic.Bool{} + cliExecutorCallback := func(event *output.InternalWrappedEvent) { + // If no results were found, and also interactsh is not being used + // in that case we can skip it, otherwise we've to show failure in + // case of matcher-status flag. + if !event.HasOperatorResult() && !event.UsesInteractsh { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + } else { + if ok := writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient); !ok { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + } + } + } + if e.options.Flow != "" { + return e.executeFlow(input, cliExecutorCallback) + } + return e.executeWithCallback(input, cliExecutorCallback) +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + userCallback := func(event *output.InternalWrappedEvent) { + if event != nil { + callback(event) + } + } + var err error + if e.options.Flow != "" { + _, err = e.executeFlow(input, userCallback) + } else { + _, err = e.executeWithCallback(input, userCallback) + } + return err +} +// executeWithCallback executes the protocol requests and calls the callback for each result. +func (e *Executer) executeWithCallback(input *contextargs.Context, callback protocols.OutputEventCallback) (bool, error) { + results := &atomic.Bool{} dynamicValues := make(map[string]interface{}) if input.HasArgs() { input.ForEach(func(key string, value interface{}) error { @@ -95,22 +136,12 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { builder.Reset() } } - // If no results were found, and also interactsh is not being used - // in that case we can skip it, otherwise we've to show failure in - // case of matcher-status flag. - if !event.HasOperatorResult() && !event.UsesInteractsh { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } - } else { - if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { - results.CompareAndSwap(false, true) - } else { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } - } + if event.HasOperatorResult() { + results.CompareAndSwap(false, true) } + // for ExecuteWithResults : this callback will execute user defined callback and some error handling + // for Execute : this callback will print the result to output + callback(event) }) if err != nil { if e.options.HostErrorsCache != nil { @@ -126,56 +157,48 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { return results.Load(), nil } -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { - dynamicValues := make(map[string]interface{}) - if input.HasArgs() { - input.ForEach(func(key string, value interface{}) error { - dynamicValues[key] = value - return nil - }) - } - previous := make(map[string]interface{}) - results := &atomic.Bool{} - +// ExecuteFlow executes template as specified in js flow +// it is mutually exclusive with executeWithCallback +func (e *Executer) executeFlow(input *contextargs.Context, callback protocols.OutputEventCallback) (bool, error) { + allprotos := make(map[string][]protocols.Request) for _, req := range e.requests { - req := req - - inputItem := input.Clone() - if e.options.InputHelper != nil && input.MetaInput.Input != "" { - if inputItem.MetaInput.Input = e.options.InputHelper.Transform(input.MetaInput.Input, req.Type()); input.MetaInput.Input == "" { - return nil - } + if req.Type() == types.MultiProtocol { + // multiprotocol execution is also mutually exclusive with flow and is slightly advanced version of executeWithCallbac() + // if request type is multiprotocol , then array does not contain any other request type + return e.executeWithCallback(input, nil) } - - err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { - ID := req.GetID() - if ID != "" { - builder := &strings.Builder{} - for k, v := range event.InternalEvent { - builder.WriteString(ID) - builder.WriteString("_") - builder.WriteString(k) - previous[builder.String()] = v - builder.Reset() - } - } - if event.OperatorsResult == nil { - return - } - results.CompareAndSwap(false, true) - callback(event) - }) - if err != nil { - if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) - } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err) - } - // If a match was found and stop at first match is set, break out of the loop and return - if results.Load() && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - break + switch req.Type() { + case types.DNSProtocol: + allprotos[types.DNSProtocol.String()] = append(allprotos[types.DNSProtocol.String()], req) + case types.HTTPProtocol: + allprotos[types.HTTPProtocol.String()] = append(allprotos[types.HTTPProtocol.String()], req) + case types.NetworkProtocol: + allprotos[types.NetworkProtocol.String()] = append(allprotos[types.NetworkProtocol.String()], req) + case types.FileProtocol: + allprotos[types.FileProtocol.String()] = append(allprotos[types.FileProtocol.String()], req) + case types.HeadlessProtocol: + allprotos[types.HeadlessProtocol.String()] = append(allprotos[types.HeadlessProtocol.String()], req) + case types.SSLProtocol: + allprotos[types.SSLProtocol.String()] = append(allprotos[types.SSLProtocol.String()], req) + case types.WebsocketProtocol: + allprotos[types.WebsocketProtocol.String()] = append(allprotos[types.WebsocketProtocol.String()], req) + case types.WHOISProtocol: + allprotos[types.WHOISProtocol.String()] = append(allprotos[types.WHOISProtocol.String()], req) + case types.CodeProtocol: + allprotos[types.CodeProtocol.String()] = append(allprotos[types.CodeProtocol.String()], req) } } - return nil + flow := &FlowExecutor{ + allProtocols: allprotos, + input: input, + options: e.options, + allErrs: mapsutil.SyncLockMap[string, error]{}, + results: &atomic.Bool{}, + } + + if err := flow.Compile(callback); err != nil { + return false, err + } + + return flow.Execute() } diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go new file mode 100644 index 0000000000..f520b943df --- /dev/null +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -0,0 +1,149 @@ +package executer + +import ( + "strconv" + "sync/atomic" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + errorutil "github.com/projectdiscovery/utils/errors" + mapsutil "github.com/projectdiscovery/utils/maps" + "go.uber.org/multierr" +) + +var ( + ErrInvalidRequestID = errorutil.NewWithFmt("invalid request id '%s' provided") +) + +type FlowExecutor struct { + input *contextargs.Context + allProtocols map[string][]protocols.Request + options *protocols.ExecutorOptions + allErrs mapsutil.SyncLockMap[string, error] + results *atomic.Bool +} + +// Init initializes the flow executor all dependencies +// this compiles and prepares for execution of a flow +// since it has dependencies on variables and etc it can't be done moved to templates package +func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent)) error { + // define variables/objects + VarRegistry := map[string]func(id ...string){} + // store all dynamic variables and other variables here + f.options.TemplateCtx = contextargs.New() + + if f.results == nil { + f.results = new(atomic.Bool) + } + + // add all input args to template context + if f.input.HasArgs() { + f.input.ForEach(func(key string, value interface{}) error { + f.options.TemplateCtx.Set(key, value) + return nil + }) + } + // variables + f.options.Variables.ForEach(func(key string, value interface{}) { + f.options.TemplateCtx.Set(key, value) + }) + // cli options + optionVars := generators.BuildPayloadFromOptions(f.options.Options) + // constants + constants := f.options.Constants + allVars := generators.MergeMaps(optionVars, constants) + // merge all variables + f.options.TemplateCtx.Merge(allVars) + + compileErrors := []error{} + + for proto, requests := range f.allProtocols { + reqMap := mapsutil.Map[string, protocols.Request]{} + counter := 0 + for index := range requests { + request := f.allProtocols[proto][index] + if request.GetID() != "" { + // if id is present use it + reqMap[request.GetID()] = request + } else { + // fallback to using index as id + reqMap[strconv.Itoa(counter)] = request + } + counter++ + } + VarRegistry[proto] = func(ids ...string) { + // if no id is passed execute all requests in sequence + if len(ids) == 0 { + // execution logic for http() + for index := range f.allProtocols[proto] { + req := f.allProtocols[proto][index] + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { + if result != nil { + f.results.CompareAndSwap(false, true) + callback(result) + // export dynamic values from operators (i.e internal:true) + // add add it to template context + if result.HasOperatorResult() && len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.TemplateCtx.Set(k, v) + } + } + } + }) + if err != nil { + // save all errors in a map with id as key + // its less likely that there will be race condition but just in case + id := req.GetID() + if id == "" { + id, _ = reqMap.GetKeyWithValue(req) + } + _ = f.allErrs.Set(id, err) + return + } + } + return + } + + // execution logic for http("0") or http("get-aws-vpcs") + for _, id := range ids { + req, ok := reqMap[id] + if !ok { + // compile error + compileErrors = append(compileErrors, ErrInvalidRequestID.Msgf(id)) + return + } + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { + if result != nil { + f.results.CompareAndSwap(false, true) + callback(result) + // export dynamic values from operators (i.e internal:true) + // add add it to template context + if result.HasOperatorResult() && len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.TemplateCtx.Set(k, v) + } + } + } + }) + if err != nil { + index := id + _ = f.allErrs.Set(index, err) + } + } + } + } + + if len(compileErrors) > 0 { + return multierr.Combine(compileErrors...) + } + + return nil +} + +// Execute executes the flow +func (f *FlowExecutor) Execute() (bool, error) { + // pass flow and execute the js vm and handle errors + return f.results.Load(), nil +} diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 6f7c3bf88a..10c4f27692 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -91,12 +91,14 @@ type ExecutorOptions struct { TemplateCtx *contextargs.Context // ProtocolType is the type of the template ProtocolType templateTypes.ProtocolType + // Flow is execution flow for the template (written in javascript) + Flow string } // AddTemplateVars adds vars to template context with given template type as prefix // this method is no-op if template is not multi protocol func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, vars map[string]interface{}) { - if e.ProtocolType != templateTypes.MultiProtocol { + if e.ProtocolType != templateTypes.MultiProtocol || e.Flow != "" { // no-op if not multi protocol template return } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 5ceea4f5e0..a387eea59f 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "reflect" + "strings" "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -238,6 +239,11 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option options.Variables = template.Variables } + // TODO: we should add a syntax check here + if strings.TrimSpace(template.Flow) != "" { + options.Flow = template.Flow + } + // create empty context args for template scope options.TemplateCtx = contextargs.New() options.ProtocolType = template.Type() diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 6d00f2a4ab..d30ab3cfd0 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -47,6 +47,18 @@ type Template struct { // - value: exampleInfoStructure Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"` // description: | + // Flow contains the execution flow for the template. + // examples: + // - flow: | + // for region in regions { + // http(0) + // } + // for vpc in vpcs { + // http(1) + // } + // + Flow string `yaml:"flow,omitempty" json:"flow,omitempty" jsonschema:"title=template execution flow in js,description=Flow contains js code which defines how the template should be executed"` + // description: | // Requests contains the http request to make in the template. // WARNING: 'requests' will be deprecated and will be removed in a future release. Please use 'http' instead. // examples: From 025023001c40abbb3f98e38ede235c207fccba23 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 2 Aug 2023 03:11:31 +0530 Subject: [PATCH 02/38] progress --- v2/cmd/nuclei/nuclei_flow.yaml | 69 +++++++++++++++++ v2/cmd/nuclei/regions.txt | 2 + v2/cmd/nuclei/test.yaml | 18 ----- v2/cmd/nuclei/vpc_logging.js | 8 ++ v2/go.mod | 4 + v2/go.sum | 23 +++++- v2/pkg/protocols/common/executer/executer.go | 5 +- .../common/executer/flow_executor.go | 77 ++++++++++++++++++- v2/pkg/templates/compile.go | 10 +++ v2/pkg/templates/templates.go | 5 ++ 10 files changed, 200 insertions(+), 21 deletions(-) create mode 100644 v2/cmd/nuclei/nuclei_flow.yaml create mode 100644 v2/cmd/nuclei/regions.txt delete mode 100644 v2/cmd/nuclei/test.yaml create mode 100644 v2/cmd/nuclei/vpc_logging.js diff --git a/v2/cmd/nuclei/nuclei_flow.yaml b/v2/cmd/nuclei/nuclei_flow.yaml new file mode 100644 index 0000000000..5ae2f90744 --- /dev/null +++ b/v2/cmd/nuclei/nuclei_flow.yaml @@ -0,0 +1,69 @@ +id: vpc-logging-not-enabled +info: + name: VPC flow logging Not enabled + author: princechaddha + severity: low + description: | + Checks if the Amazon CloudWatch alarmed is enabled for Sign In without MFA + reference: + - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeVpcs.html + - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeFlowLogs.html + remediation: Enabling VPC Flow Logs + tags: cloud,devops,aws,amazon,vpc,ec2,logging + +self-contained: true +signature: aws +variables: + service: ec2 + regions: regions.txt + +flow: + vpc_logging.js + +http: + - id: extract-vpcs + raw: + - | + POST https://{{service}}.{{region}}.amazonaws.com/ HTTP/1.1 + Host: {{service}}.{{region}}.amazonaws.com + Content-Type: application/x-www-form-urlencoded; charset=utf-8 + User-Agent: aws-cli/2.9.15 Python/3.11.1 Darwin/22.3.0 source/arm64 prompt/off command/ec2.describe-vpcs + + Action=DescribeVpcs&Version=2016-11-15 + + extractors: + - type: regex + name: vpcs + part: body + internal: true + group: 1 + regex: + - '([a-z-0-9]+)<\/vpcId>' + + - id: extract-flow-logs + raw: + - | + POST https://{{service}}.{{region}}.amazonaws.com/ HTTP/1.1 + Host: {{service}}.{{region}}.amazonaws.com + Content-Type: application/x-www-form-urlencoded; charset=utf-8 + User-Agent: aws-cli/2.9.15 Python/3.11.1 Darwin/22.3.0 source/arm64 prompt/off command/ec2.describe-flow-logs + + Action=DescribeFlowLogs&Version=2016-11-15&Filter.1.Name=resource-id&Filter.1.Value.1={{vpcId}} + + iterate-all: true + matchers-condition: and + matchers: + - type: word + part: body + words: + - "" + + - type: word + words: + - "" + negative: true + + extractors: + - type: dsl + dsl: + - '"VPC flow logging is not enabled for " + vpcId + " in " + region + " region"' \ No newline at end of file diff --git a/v2/cmd/nuclei/regions.txt b/v2/cmd/nuclei/regions.txt new file mode 100644 index 0000000000..cd0800332e --- /dev/null +++ b/v2/cmd/nuclei/regions.txt @@ -0,0 +1,2 @@ +us-east-1 +us-west-1 diff --git a/v2/cmd/nuclei/test.yaml b/v2/cmd/nuclei/test.yaml deleted file mode 100644 index ecb62d2ec6..0000000000 --- a/v2/cmd/nuclei/test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -id: basic-example - -info: - name: Test HTTP Template - author: pdteam - severity: info - -http: - - raw: - - |+ - GET / HTTP/1.1 - Host: {{Hostname}} - - unsafe: true - matchers: - - type: dsl - dsl: - - true \ No newline at end of file diff --git a/v2/cmd/nuclei/vpc_logging.js b/v2/cmd/nuclei/vpc_logging.js new file mode 100644 index 0000000000..3daaf44a72 --- /dev/null +++ b/v2/cmd/nuclei/vpc_logging.js @@ -0,0 +1,8 @@ +log(template) + +template["regions"].forEach(element => { + http("extract-vpcs") +}); + +log("got vpcs") +log(template["vpcs"]) \ No newline at end of file diff --git a/v2/go.mod b/v2/go.mod index cc2fd428ac..8cbc1b34f5 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -60,6 +60,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67 github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 github.com/docker/go-units v0.5.0 + github.com/dop251/goja v0.0.0-20230707174833-636fdf960de1 github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.7.0 github.com/h2non/filetype v1.1.3 @@ -109,8 +110,10 @@ require ( github.com/fatih/color v1.14.1 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gaukas/godicttls v0.0.3 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/google/certificate-transparency-go v1.1.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect + github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect @@ -133,6 +136,7 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect github.com/smartystreets/assertions v1.0.0 // indirect + github.com/smartystreets/goconvey v1.6.4 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect github.com/tidwall/gjson v1.14.4 // indirect diff --git a/v2/go.sum b/v2/go.sum index 31228c2af1..3ab8eaf4df 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -143,6 +143,9 @@ github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM2 github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= @@ -160,11 +163,18 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20230707174833-636fdf960de1 h1:sC/DYk3eEi5cKkpJX1vl+CpAM138dmuW7rutje9Eo4E= +github.com/dop251/goja v0.0.0-20230707174833-636fdf960de1/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= @@ -205,6 +215,8 @@ github.com/go-playground/validator/v10 v10.14.1 h1:9c50NUPC30zyuKprjL3vNZ0m5oG+j github.com/go-playground/validator/v10 v10.14.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-rod/rod v0.113.0 h1:E7+GLjYVZnScewIB2u8+66joQLaDGbOLzSOT4orNHms= github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -249,6 +261,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= +github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -274,6 +288,7 @@ github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDj github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU= @@ -310,6 +325,7 @@ github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -465,6 +481,7 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -491,8 +508,9 @@ github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2Iqp github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= @@ -703,6 +721,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -732,6 +751,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -775,6 +795,7 @@ gopkg.in/corvus-ch/zbase32.v1 v1.0.0 h1:K4u1NprbDNvKPczKfHLbwdOWHTZ0zfv2ow71H1nR gopkg.in/corvus-ch/zbase32.v1 v1.0.0/go.mod h1:T3oKkPOm4AV/bNXCNFUxRmlE9RUyBz/DSo0nK9U+c0Y= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 248dac81c5..6a05e580c0 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + errorutil "github.com/projectdiscovery/utils/errors" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -87,6 +88,7 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + gologger.Info().Msgf("[%s] Running on %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint()) userCallback := func(event *output.InternalWrappedEvent) { if event != nil { callback(event) @@ -165,6 +167,7 @@ func (e *Executer) executeFlow(input *contextargs.Context, callback protocols.Ou if req.Type() == types.MultiProtocol { // multiprotocol execution is also mutually exclusive with flow and is slightly advanced version of executeWithCallbac() // if request type is multiprotocol , then array does not contain any other request type + gologger.Info().Msgf("reqtype is %v", req.Type().String()) return e.executeWithCallback(input, nil) } switch req.Type() { @@ -197,7 +200,7 @@ func (e *Executer) executeFlow(input *contextargs.Context, callback protocols.Ou } if err := flow.Compile(callback); err != nil { - return false, err + return false, errorutil.NewWithErr(err).Msgf("could not compile flow") } return flow.Execute() diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index f520b943df..0b83d6287c 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -1,14 +1,20 @@ package executer import ( + "fmt" + "io" "strconv" + "strings" "sync/atomic" + "github.com/dop251/goja" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps" "go.uber.org/multierr" ) @@ -23,6 +29,8 @@ type FlowExecutor struct { options *protocols.ExecutorOptions allErrs mapsutil.SyncLockMap[string, error] results *atomic.Bool + jsVM *goja.Runtime + program *goja.Program } // Init initializes the flow executor all dependencies @@ -54,8 +62,17 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) // constants constants := f.options.Constants allVars := generators.MergeMaps(optionVars, constants) - // merge all variables f.options.TemplateCtx.Merge(allVars) + // TODO: this is another sandbox bypass we only expand in generators.go + // but we require it now in multiple places (move sandbox and payload read logic to config.DefaultConfig()) + // merge all variables + for k, v := range f.options.TemplateCtx.GetAll() { + if str, ok := v.(string); ok && len(str) < 150 && fileutil.FileExists(str) { + if value, err := f.ReadDataFromFile(str); err == nil { + f.options.TemplateCtx.Set(k, value) + } + } + } compileErrors := []error{} @@ -124,6 +141,8 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) for k, v := range result.OperatorsResult.DynamicValues { f.options.TemplateCtx.Set(k, v) } + f.jsVM.Set("template", f.options.TemplateCtx.GetAll()) + f.jsVM.Set("values", "adding new var in execution") } } }) @@ -139,11 +158,67 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) return multierr.Combine(compileErrors...) } + // create a new js vm/runtime + f.jsVM = goja.New() + if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value { + arg := call.Argument(0) + fmt.Println("log called with", arg.Export()) + return goja.Null() + }); err != nil { + return err + } + + var m map[string]interface{} = f.options.TemplateCtx.GetAll() + + if err := f.jsVM.Set("template", m); err != nil { + // all template variables are available in js template object + return err + } + + // register all protocols + for name, fn := range VarRegistry { + if err := f.jsVM.Set(name, fn); err != nil { + return err + } + } + + program, err := goja.Compile("flow", f.options.Flow, false) + if err != nil { + return err + } + f.program = program + return nil } // Execute executes the flow func (f *FlowExecutor) Execute() (bool, error) { // pass flow and execute the js vm and handle errors + value, err := f.jsVM.RunProgram(f.program) + if err != nil { + return false, errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) + } + gologger.Verbose().Msgf("Flow result: %s", value.String()) + return f.results.Load(), nil } + +func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { + values := []string{} + reader, err := f.options.Catalog.OpenFile(payload) + if err != nil { + return values, err + } + defer reader.Close() + bin, err := io.ReadAll(reader) + if err != nil { + return values, err + } + for _, line := range strings.Split(string(bin), "\n") { + line = strings.TrimSpace(line) + if line != "" { + values = append(values, line) + } + } + return values, nil +} diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index a387eea59f..12c7370047 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -3,6 +3,8 @@ package templates import ( "fmt" "io" + "os" + "path/filepath" "reflect" "strings" @@ -18,6 +20,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/retryablehttp-go" + fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -241,6 +244,13 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option // TODO: we should add a syntax check here if strings.TrimSpace(template.Flow) != "" { + if len(template.Flow) > 0 && filepath.Ext(template.Flow) == ".js" && fileutil.FileExists(template.Flow) { + // TODO: this is sandbox bypass, we should remove it + // move sandbox check to config so it can be reused here and in the generator + if bin, err := os.ReadFile(template.Flow); err == nil { + template.Flow = string(bin) + } + } options.Flow = template.Flow } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index d30ab3cfd0..f0ab891539 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -264,6 +264,11 @@ func (template *Template) addProtocolsToQueue(keys ...string) { // isMultiProtocol checks if the template is a multi protocol template func (template *Template) isMultiProtocol() bool { + // Review: if template contains more than 1 protocol request (doesn't matter if it is same protocol or different) + // it is considered as multi protocol template + if template.Flow != "" { + return false + } counter := len(template.RequestsDNS) + len(template.RequestsFile) + len(template.RequestsHTTP) + len(template.RequestsHeadless) + len(template.RequestsNetwork) + len(template.RequestsSSL) + From d9bddd56dae760f43dacf255020eba77354d54cc Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 2 Aug 2023 04:27:12 +0530 Subject: [PATCH 03/38] working POC --- v2/cmd/nuclei/regions.txt | 2 ++ v2/cmd/nuclei/vpc_logging.js | 20 +++++++++++--- .../common/contextargs/contextargs.go | 21 +++++++++++++++ .../common/executer/flow_executor.go | 27 ++++++++++++++++--- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/v2/cmd/nuclei/regions.txt b/v2/cmd/nuclei/regions.txt index cd0800332e..9465475f6b 100644 --- a/v2/cmd/nuclei/regions.txt +++ b/v2/cmd/nuclei/regions.txt @@ -1,2 +1,4 @@ us-east-1 +us-east-2 us-west-1 +us-west-2 \ No newline at end of file diff --git a/v2/cmd/nuclei/vpc_logging.js b/v2/cmd/nuclei/vpc_logging.js index 3daaf44a72..f94087c23f 100644 --- a/v2/cmd/nuclei/vpc_logging.js +++ b/v2/cmd/nuclei/vpc_logging.js @@ -1,8 +1,20 @@ -log(template) +log("Regions: "+template["regions"]) // just like console.log . currently does not support formatting template["regions"].forEach(element => { + set("region", element) // set variable in template context http("extract-vpcs") -}); -log("got vpcs") -log(template["vpcs"]) \ No newline at end of file + poll() // updates 'template' variable with latest data + + log("Got VPCs: "+template["vpcs"]+" from region: "+element) + + template["vpcs"].forEach(element => { + set("vpcId", element) // set variable in template context + http("extract-flow-logs") + }); +}); +// log("Got VPCs: "+template["vpcs"]+" from all regions") +// template["vpcs"].forEach(element => { +// set("vpcId", element) // set variable in template context +// http("extract-flow-logs") +// }); \ No newline at end of file diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go index 787f9583da..a0275fa111 100644 --- a/v2/pkg/protocols/common/contextargs/contextargs.go +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -41,6 +41,27 @@ func (ctx *Context) Set(key string, value interface{}) { } } +// Add the specific key-value pair +func (ctx *Context) Add(key string, v interface{}) { + values, ok := ctx.args.Get(key) + if !ok { + ctx.Set(key, v) + } + switch v := v.(type) { + case []string: + if values, ok := values.([]string); ok { + values = append(values, v...) + ctx.Set(key, values) + } + case string: + if values, ok := values.(string); ok { + tmp := []string{values, v} + ctx.Set(key, tmp) + } + } + // FixME: handle other edge cases +} + // Get the value with specific key if exists func (ctx *Context) Get(key string) (interface{}, bool) { return ctx.args.Get(key) diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 0b83d6287c..bb95e5871e 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -1,13 +1,13 @@ package executer import ( - "fmt" "io" "strconv" "strings" "sync/atomic" "github.com/dop251/goja" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" @@ -127,6 +127,7 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) for _, id := range ids { req, ok := reqMap[id] if !ok { + gologger.Error().Msgf("invalid request id '%s' provided", id) // compile error compileErrors = append(compileErrors, ErrInvalidRequestID.Msgf(id)) return @@ -142,7 +143,6 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) f.options.TemplateCtx.Set(k, v) } f.jsVM.Set("template", f.options.TemplateCtx.GetAll()) - f.jsVM.Set("values", "adding new var in execution") } } }) @@ -161,8 +161,27 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) // create a new js vm/runtime f.jsVM = goja.New() if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value { - arg := call.Argument(0) - fmt.Println("log called with", arg.Export()) + arg := call.Argument(0).Export() + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), arg) + return goja.Null() + }); err != nil { + return err + } + + if err := f.jsVM.Set("poll", func(call goja.FunctionCall) goja.Value { + var m map[string]interface{} = f.options.TemplateCtx.GetAll() + f.jsVM.Set("template", m) + return goja.Null() + }); err != nil { + return err + } + + if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value { + varName := call.Argument(0).Export() + varValue := call.Argument(1).Export() + f.options.TemplateCtx.Set(varName.(string), varValue) + // gologger.Debug().Msgf("JS: set %s to %s", varName, varValue) + // fmt.Printf("log: set %s to %s\n", varName, varValue) return goja.Null() }); err != nil { return err From fdf2eb6eb19dbee816885f5c2502d89203e59291 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 2 Aug 2023 20:31:38 +0530 Subject: [PATCH 04/38] fix string slice normalization issue in variables --- v2/cmd/nuclei/nuclei_flow.yaml | 18 +++++++++++++++--- v2/cmd/nuclei/regions.txt | 3 +-- v2/cmd/nuclei/vpc_log.js | 9 +++++++++ v2/cmd/nuclei/vpc_logging.js | 7 ++++++- .../common/executer/flow_executor.go | 5 +---- .../protocols/common/variables/variables.go | 19 +++++++++++++++++++ v2/pkg/utils/insertion_ordered_map.go | 4 +++- 7 files changed, 54 insertions(+), 11 deletions(-) create mode 100644 v2/cmd/nuclei/vpc_log.js diff --git a/v2/cmd/nuclei/nuclei_flow.yaml b/v2/cmd/nuclei/nuclei_flow.yaml index 5ae2f90744..47f2467283 100644 --- a/v2/cmd/nuclei/nuclei_flow.yaml +++ b/v2/cmd/nuclei/nuclei_flow.yaml @@ -15,10 +15,22 @@ self-contained: true signature: aws variables: service: ec2 - regions: regions.txt + regions: + - us-east-1 + - us-west-1 + - ap-south-1 -flow: - vpc_logging.js +flow: | + template["regions"].forEach(region => { + set("region",region); + http("extract-vpcs"); + poll(); + template["vpcs"].forEach(vpcId => { + set("vpcId",vpcId); + http("extract-flow-logs"); + }) + }); + http: - id: extract-vpcs diff --git a/v2/cmd/nuclei/regions.txt b/v2/cmd/nuclei/regions.txt index 9465475f6b..dc4c6be0b4 100644 --- a/v2/cmd/nuclei/regions.txt +++ b/v2/cmd/nuclei/regions.txt @@ -1,4 +1,3 @@ us-east-1 -us-east-2 us-west-1 -us-west-2 \ No newline at end of file +ap-south-1 \ No newline at end of file diff --git a/v2/cmd/nuclei/vpc_log.js b/v2/cmd/nuclei/vpc_log.js new file mode 100644 index 0000000000..a54c275c05 --- /dev/null +++ b/v2/cmd/nuclei/vpc_log.js @@ -0,0 +1,9 @@ +template["regions"].forEach(region => { + set("region",region); + http("extract-vpcs"); + poll(); + template["vpcs"].forEach(vpcId => { + set("vpcId",vpcId); + http("extract-flow-logs"); + }) +}); \ No newline at end of file diff --git a/v2/cmd/nuclei/vpc_logging.js b/v2/cmd/nuclei/vpc_logging.js index f94087c23f..6eedca261d 100644 --- a/v2/cmd/nuclei/vpc_logging.js +++ b/v2/cmd/nuclei/vpc_logging.js @@ -17,4 +17,9 @@ template["regions"].forEach(element => { // template["vpcs"].forEach(element => { // set("vpcId", element) // set variable in template context // http("extract-flow-logs") -// }); \ No newline at end of file +// }); + +template["regions"].forEach(element => { + set("region", element) // set variable in template context + http(); +}); \ No newline at end of file diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index bb95e5871e..7310579291 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -53,10 +53,7 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) return nil }) } - // variables - f.options.Variables.ForEach(func(key string, value interface{}) { - f.options.TemplateCtx.Set(key, value) - }) + f.options.TemplateCtx.Merge(f.options.Variables.GetAll()) // cli options optionVars := generators.BuildPayloadFromOptions(f.options.Options) // constants diff --git a/v2/pkg/protocols/common/variables/variables.go b/v2/pkg/protocols/common/variables/variables.go index b806402aaa..7a7fda1f03 100644 --- a/v2/pkg/protocols/common/variables/variables.go +++ b/v2/pkg/protocols/common/variables/variables.go @@ -66,6 +66,11 @@ func (variables *Variable) UnmarshalJSON(data []byte) error { func (variables *Variable) Evaluate(values map[string]interface{}) map[string]interface{} { result := make(map[string]interface{}, variables.Len()) variables.ForEach(func(key string, value interface{}) { + if sliceValue, ok := value.([]interface{}); ok { + // slices cannot be evaluated + result[key] = sliceValue + return + } valueString := types.ToString(value) combined := generators.MergeMaps(values, result) if value, ok := combined[key]; ok { @@ -76,12 +81,26 @@ func (variables *Variable) Evaluate(values map[string]interface{}) map[string]in return result } +// GetAll returns all variables as a map +func (variables *Variable) GetAll() map[string]interface{} { + result := make(map[string]interface{}, variables.Len()) + variables.ForEach(func(key string, value interface{}) { + result[key] = value + }) + return result +} + // EvaluateWithInteractsh returns evaluation results of variables with interactsh func (variables *Variable) EvaluateWithInteractsh(values map[string]interface{}, interact *interactsh.Client) (map[string]interface{}, []string) { result := make(map[string]interface{}, variables.Len()) var interactURLs []string variables.ForEach(func(key string, value interface{}) { + if sliceValue, ok := value.([]interface{}); ok { + // slices cannot be evaluated + result[key] = sliceValue + return + } valueString := types.ToString(value) if strings.Contains(valueString, "interactsh-url") { valueString, interactURLs = interact.Replace(valueString, interactURLs) diff --git a/v2/pkg/utils/insertion_ordered_map.go b/v2/pkg/utils/insertion_ordered_map.go index a9586778af..0e3b495d9b 100644 --- a/v2/pkg/utils/insertion_ordered_map.go +++ b/v2/pkg/utils/insertion_ordered_map.go @@ -58,7 +58,7 @@ func (insertionOrderedStringMap *InsertionOrderedStringMap) UnmarshalJSON(data [ } // toString converts an interface to string in a quick way -func toString(data interface{}) string { +func toString(data interface{}) interface{} { switch s := data.(type) { case nil: return "" @@ -92,6 +92,8 @@ func toString(data interface{}) string { return strconv.FormatUint(uint64(s), 10) case []byte: return string(s) + case []interface{}: + return data default: return fmt.Sprintf("%v", data) } From cffe9ce707715febfa21f80d2d8c2ff9aad956cc Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 3 Aug 2023 00:55:53 +0530 Subject: [PATCH 05/38] update --- v2/cmd/nuclei/nuclei_flow.yaml | 81 ------------------- v2/cmd/nuclei/vpc_log.js | 9 --- v2/cmd/nuclei/vpc_logging.js | 25 ------ .../common/executer/flow_executor.go | 1 + v2/pkg/protocols/http/build_request.go | 3 +- v2/pkg/protocols/http/request.go | 1 + 6 files changed, 4 insertions(+), 116 deletions(-) delete mode 100644 v2/cmd/nuclei/nuclei_flow.yaml delete mode 100644 v2/cmd/nuclei/vpc_log.js delete mode 100644 v2/cmd/nuclei/vpc_logging.js diff --git a/v2/cmd/nuclei/nuclei_flow.yaml b/v2/cmd/nuclei/nuclei_flow.yaml deleted file mode 100644 index 47f2467283..0000000000 --- a/v2/cmd/nuclei/nuclei_flow.yaml +++ /dev/null @@ -1,81 +0,0 @@ -id: vpc-logging-not-enabled -info: - name: VPC flow logging Not enabled - author: princechaddha - severity: low - description: | - Checks if the Amazon CloudWatch alarmed is enabled for Sign In without MFA - reference: - - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeVpcs.html - - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeFlowLogs.html - remediation: Enabling VPC Flow Logs - tags: cloud,devops,aws,amazon,vpc,ec2,logging - -self-contained: true -signature: aws -variables: - service: ec2 - regions: - - us-east-1 - - us-west-1 - - ap-south-1 - -flow: | - template["regions"].forEach(region => { - set("region",region); - http("extract-vpcs"); - poll(); - template["vpcs"].forEach(vpcId => { - set("vpcId",vpcId); - http("extract-flow-logs"); - }) - }); - - -http: - - id: extract-vpcs - raw: - - | - POST https://{{service}}.{{region}}.amazonaws.com/ HTTP/1.1 - Host: {{service}}.{{region}}.amazonaws.com - Content-Type: application/x-www-form-urlencoded; charset=utf-8 - User-Agent: aws-cli/2.9.15 Python/3.11.1 Darwin/22.3.0 source/arm64 prompt/off command/ec2.describe-vpcs - - Action=DescribeVpcs&Version=2016-11-15 - - extractors: - - type: regex - name: vpcs - part: body - internal: true - group: 1 - regex: - - '([a-z-0-9]+)<\/vpcId>' - - - id: extract-flow-logs - raw: - - | - POST https://{{service}}.{{region}}.amazonaws.com/ HTTP/1.1 - Host: {{service}}.{{region}}.amazonaws.com - Content-Type: application/x-www-form-urlencoded; charset=utf-8 - User-Agent: aws-cli/2.9.15 Python/3.11.1 Darwin/22.3.0 source/arm64 prompt/off command/ec2.describe-flow-logs - - Action=DescribeFlowLogs&Version=2016-11-15&Filter.1.Name=resource-id&Filter.1.Value.1={{vpcId}} - - iterate-all: true - matchers-condition: and - matchers: - - type: word - part: body - words: - - "" - - - type: word - words: - - "" - negative: true - - extractors: - - type: dsl - dsl: - - '"VPC flow logging is not enabled for " + vpcId + " in " + region + " region"' \ No newline at end of file diff --git a/v2/cmd/nuclei/vpc_log.js b/v2/cmd/nuclei/vpc_log.js deleted file mode 100644 index a54c275c05..0000000000 --- a/v2/cmd/nuclei/vpc_log.js +++ /dev/null @@ -1,9 +0,0 @@ -template["regions"].forEach(region => { - set("region",region); - http("extract-vpcs"); - poll(); - template["vpcs"].forEach(vpcId => { - set("vpcId",vpcId); - http("extract-flow-logs"); - }) -}); \ No newline at end of file diff --git a/v2/cmd/nuclei/vpc_logging.js b/v2/cmd/nuclei/vpc_logging.js deleted file mode 100644 index 6eedca261d..0000000000 --- a/v2/cmd/nuclei/vpc_logging.js +++ /dev/null @@ -1,25 +0,0 @@ -log("Regions: "+template["regions"]) // just like console.log . currently does not support formatting - -template["regions"].forEach(element => { - set("region", element) // set variable in template context - http("extract-vpcs") - - poll() // updates 'template' variable with latest data - - log("Got VPCs: "+template["vpcs"]+" from region: "+element) - - template["vpcs"].forEach(element => { - set("vpcId", element) // set variable in template context - http("extract-flow-logs") - }); -}); -// log("Got VPCs: "+template["vpcs"]+" from all regions") -// template["vpcs"].forEach(element => { -// set("vpcId", element) // set variable in template context -// http("extract-flow-logs") -// }); - -template["regions"].forEach(element => { - set("region", element) // set variable in template context - http(); -}); \ No newline at end of file diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 7310579291..e2f3713e84 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -99,6 +99,7 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) callback(result) // export dynamic values from operators (i.e internal:true) // add add it to template context + // this is a conflicting behaviour with iterate-all if result.HasOperatorResult() && len(result.OperatorsResult.DynamicValues) > 0 { for k, v := range result.OperatorsResult.DynamicValues { f.options.TemplateCtx.Set(k, v) diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index c82c13d40d..afd80e8315 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -72,7 +72,8 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, // 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path // add template context values to dynamicValues (this takes care of self-contained and other types of requests) - dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.TemplateCtx.GetAll()) + // dynamicValues should be given preference over templateCtx values + dynamicValues = generators.MergeMaps(r.request.options.TemplateCtx.GetAll(), dynamicValues) if r.request.SelfContained { return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index fec2708231..98989fc54c 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -426,6 +426,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa var gotErr error var skip bool if len(gotDynamicValues) > 0 { + // fmt.Printf("dynamic values %v and templateCtx values %v\n", gotDynamicValues, request.options.TemplateCtx.GetAll()) operators.MakeDynamicValuesCallback(gotDynamicValues, request.IterateAll, func(data map[string]interface{}) bool { if skip, gotErr = executeFunc(inputData, payloads, data); skip || gotErr != nil { return true From 14012ab5a85f5b6deac84cbfccef22dc102196c5 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 3 Aug 2023 19:19:43 +0530 Subject: [PATCH 06/38] fix nil panic --- v2/pkg/protocols/common/executer/executer.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 6a05e580c0..6204d7fe66 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -195,8 +195,11 @@ func (e *Executer) executeFlow(input *contextargs.Context, callback protocols.Ou allProtocols: allprotos, input: input, options: e.options, - allErrs: mapsutil.SyncLockMap[string, error]{}, - results: &atomic.Bool{}, + allErrs: mapsutil.SyncLockMap[string, error]{ + ReadOnly: atomic.Bool{}, + Map: make(map[string]error), + }, + results: &atomic.Bool{}, } if err := flow.Compile(callback); err != nil { From 1bca35fad239199c4bdf36ae9a914f6b95218be8 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 3 Aug 2023 19:47:46 +0530 Subject: [PATCH 07/38] remove poll() --- v2/pkg/protocols/common/executer/flow_executor.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index e2f3713e84..d4f1543e3e 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -46,6 +46,9 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) f.results = new(atomic.Bool) } + // create a new js vm/runtime + f.jsVM = goja.New() + // add all input args to template context if f.input.HasArgs() { f.input.ForEach(func(key string, value interface{}) error { @@ -88,6 +91,10 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) counter++ } VarRegistry[proto] = func(ids ...string) { + defer func() { + var m map[string]interface{} = f.options.TemplateCtx.GetAll() + f.jsVM.Set("template", m) + }() // if no id is passed execute all requests in sequence if len(ids) == 0 { // execution logic for http() @@ -156,8 +163,6 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) return multierr.Combine(compileErrors...) } - // create a new js vm/runtime - f.jsVM = goja.New() if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value { arg := call.Argument(0).Export() gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), arg) From 6099997b6eacdab8f66bfea058e815e5741fab30 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 7 Aug 2023 21:42:21 +0530 Subject: [PATCH 08/38] load file with sandbox and more --- .../protocols/code/py-env-var.yaml | 2 +- integration_tests/protocols/code/py-file.yaml | 2 +- .../protocols/code/py-interactsh.yaml | 2 +- .../protocols/code/py-snippet.yaml | 2 +- .../protocols/http/matcher-status.yaml | 2 +- v2/cmd/integration-test/template-path.go | 2 +- v2/pkg/catalog/config/constants.go | 2 +- .../common/contextargs/contextargs.go | 6 +- v2/pkg/protocols/common/executer/executer.go | 4 +- .../common/executer/flow_executor.go | 88 ++++++++------- .../protocols/common/generators/generators.go | 9 +- .../common/generators/generators_test.go | 13 ++- v2/pkg/protocols/common/generators/load.go | 26 ++--- .../protocols/common/generators/load_test.go | 101 +++++++++++++----- v2/pkg/protocols/dns/dns.go | 2 +- v2/pkg/protocols/headless/headless.go | 2 +- v2/pkg/protocols/http/build_request.go | 4 +- v2/pkg/protocols/http/http.go | 2 +- v2/pkg/protocols/http/request.go | 1 - .../protocols/http/request_generator_test.go | 5 +- v2/pkg/protocols/network/network.go | 2 +- v2/pkg/protocols/protocols.go | 7 +- v2/pkg/protocols/websocket/websocket.go | 2 +- v2/pkg/templates/compile.go | 17 ++- v2/pkg/types/types.go | 31 ++++++ 25 files changed, 220 insertions(+), 116 deletions(-) diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 0e5c9c2881..2e3a3a6194 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4b0a00483046022100cd2b9d34169cdb716caee25976fed763880435f2f1e2979c9d7c9d2bd7b8e409022100dd0ba8bd3fa6a6be5f964ca3b0ce8bdbb20d865553133cf494ef64fbeebff345 \ No newline at end of file +# digest: 490a00463044022013021390d00638aa2c178f386cd45a9bbd560ec19ed834755d1e2379b967dcdd0220535a7763e78a16fa149a8a04ebeb25d9124a046e39810d828ba37a1ea1109066 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 8fae1b051a..7af0b3f214 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a00483046022100f663e5afaf5c118b21b9c5918cba12d7cc83edc2a3ee0f338c07e3cd1fe40e20022100b46193e3275c490a4ad3897c6e2ca51ce09f408538b17d041e0063d40f4df833 \ No newline at end of file +# digest: 490a0046304402203269b5ac13def28e907e7529863b92c1fb4b8e20328c284a4ed99bf5d871505f02200fb8b8ed0b6a8dc00dc999d9acc9292681a34166c3c760411ac764fd343d0c0a \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index de4abdac08..af4dfba02f 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4b0a00483046022100c45cd27b9d49879663e1ea3c877dc362d06b8a0aea64b1ab06be3af5aa9a32ee0221008f5ee347245a2c1e04c46528e4c70a5a851f95c6ba49d2834ef7c3784bca47a9 \ No newline at end of file +# digest: 490a004630440220740d6c3ac20692f051b7642c7ffed00ff4b804d60110b067a7083e56bdcae17502202a4d7256e9a10a717556ff44de55dceeb1521afe9cda8955ea80dc401495c259 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 9b0187dd8d..51aa8d19be 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a00473045022100df57bf446d6d8e73ff9424b1055faebcea9038e5d5934834ed8e619b77bdfd5e02201754c1cebe9f65883315b3830755a0689999f33db7102cd8d5469e4c01cc6a66 \ No newline at end of file +# digest: 4a0a00473045022100bfc50dec85d30f8ce57bf2a2ca22d9759e1045426781bdbced085bd42c4db9db02201cc3510758ce06c50976910919a71a2310564ad1815bffd55e0864335a4735f0 \ No newline at end of file diff --git a/integration_tests/protocols/http/matcher-status.yaml b/integration_tests/protocols/http/matcher-status.yaml index 5704c2a3d9..4cfd0d1a0b 100644 --- a/integration_tests/protocols/http/matcher-status.yaml +++ b/integration_tests/protocols/http/matcher-status.yaml @@ -1,4 +1,4 @@ -id: matchet-status +id: matcher-status info: name: Test Matcher Status diff --git a/v2/cmd/integration-test/template-path.go b/v2/cmd/integration-test/template-path.go index 773f919254..a4e483a405 100644 --- a/v2/cmd/integration-test/template-path.go +++ b/v2/cmd/integration-test/template-path.go @@ -16,7 +16,7 @@ var templatesPathTestCases = []TestCaseInfo{ //template folder path issue {Path: "protocols/http/get.yaml", TestCase: &folderPathTemplateTest{}}, //cwd - {Path: "./dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, + {Path: "./protocols/dns/cname-fingerprint.yaml", TestCase: &cwdTemplateTest{}}, //relative path {Path: "dns/dns-saas-service-detection.yaml", TestCase: &relativePathTemplateTest{}}, //absolute path diff --git a/v2/pkg/catalog/config/constants.go b/v2/pkg/catalog/config/constants.go index 8b945c4ad6..2f6b4fbff8 100644 --- a/v2/pkg/catalog/config/constants.go +++ b/v2/pkg/catalog/config/constants.go @@ -17,7 +17,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.0.0` + Version = `v3.0.0-dev` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go index da117a8f14..9b34457c67 100644 --- a/v2/pkg/protocols/common/contextargs/contextargs.go +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -61,6 +61,8 @@ func (ctx *Context) Add(key string, v interface{}) { if !ok { ctx.Set(key, v) } + + // If the key exists, append the value to the existing value switch v := v.(type) { case []string: if values, ok := values.([]string); ok { @@ -72,8 +74,10 @@ func (ctx *Context) Add(key string, v interface{}) { tmp := []string{values, v} ctx.Set(key, tmp) } + default: + values, _ := ctx.Get(key) + ctx.Set(key, []interface{}{values, v}) } - // FixME: handle other edge cases } // Get the value with specific key if exists diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 949043606f..4b5d2b250b 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -97,11 +97,11 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { if e.options.Flow != "" { return e.executeFlow(input, cliExecutorCallback) } - ok, err := e.executeWithCallback(input, results, cliExecutorCallback) + _, err := e.executeWithCallback(input, results, cliExecutorCallback) if lastMatcherEvent != nil { writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) } - return ok, err + return results.Load(), err } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 7de2684ad3..ff7209c0f2 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -24,55 +24,56 @@ var ( ) type FlowExecutor struct { - input *contextargs.Context - allProtocols map[string][]protocols.Request - options *protocols.ExecutorOptions - allErrs mapsutil.SyncLockMap[string, error] - results *atomic.Bool - jsVM *goja.Runtime - program *goja.Program + input *contextargs.Context + allProtocols map[string][]protocols.Request + options *protocols.ExecutorOptions + allErrs mapsutil.SyncLockMap[string, error] + results *atomic.Bool + jsVM *goja.Runtime + program *goja.Program + protoFunctions map[string]func(id ...string) // reqFunctions contains functions that allow executing requests/protocols from js } // Init initializes the flow executor all dependencies // this compiles and prepares for execution of a flow // since it has dependencies on variables and etc it can't be done moved to templates package func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent)) error { - // define variables/objects - VarRegistry := map[string]func(id ...string){} - // store all dynamic variables and other variables here - f.options.TemplateCtx = contextargs.New() - if f.results == nil { f.results = new(atomic.Bool) } - + // store all dynamic variables and other variables here + f.options.TemplateCtx = contextargs.New() // create a new js vm/runtime f.jsVM = goja.New() + // -----Load all types of variables----- // add all input args to template context if f.input.HasArgs() { f.input.ForEach(func(key string, value interface{}) { f.options.TemplateCtx.Set(key, value) }) } - f.options.TemplateCtx.Merge(f.options.Variables.GetAll()) + // load all variables and evaluate with existing data + variableMap := f.options.Variables.Evaluate(f.options.TemplateCtx.GetAll()) // cli options optionVars := generators.BuildPayloadFromOptions(f.options.Options) // constants constants := f.options.Constants - allVars := generators.MergeMaps(optionVars, constants) - f.options.TemplateCtx.Merge(allVars) - // TODO: this is another sandbox bypass we only expand in generators.go - // but we require it now in multiple places (move sandbox and payload read logic to config.DefaultConfig()) - // merge all variables - for k, v := range f.options.TemplateCtx.GetAll() { + allVars := generators.MergeMaps(variableMap, constants, optionVars) + // we support loading variables from files in variables , cli options and constants + // try to load if files exist + for k, v := range allVars { if str, ok := v.(string); ok && len(str) < 150 && fileutil.FileExists(str) { if value, err := f.ReadDataFromFile(str); err == nil { - f.options.TemplateCtx.Set(k, value) + allVars[k] = value } } } + f.options.TemplateCtx.Merge(allVars) + // ------ + // ---- define callback functions/objects---- + f.protoFunctions = map[string]func(id ...string){} compileErrors := []error{} for proto, requests := range f.allProtocols { @@ -83,20 +84,22 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) if request.GetID() != "" { // if id is present use it reqMap[request.GetID()] = request - } else { - // fallback to using index as id - reqMap[strconv.Itoa(counter)] = request } + // fallback to using index as id + // always allow index as id as a fallback + reqMap[strconv.Itoa(counter)] = request counter++ } - VarRegistry[proto] = func(ids ...string) { + // ---define hook that allows protocol/request execution from js----- + f.protoFunctions[proto] = func(ids ...string) { defer func() { + // to avoid polling update template variables everytime we execute a protocol var m map[string]interface{} = f.options.TemplateCtx.GetAll() _ = f.jsVM.Set("template", m) }() // if no id is passed execute all requests in sequence if len(ids) == 0 { - // execution logic for http() + // execution logic for http()/dns() etc for index := range f.allProtocols[proto] { req := f.allProtocols[proto][index] err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { @@ -162,7 +165,23 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) return multierr.Combine(compileErrors...) } + // register all built in functions + return f.RegisterBuiltInFunctions() +} + +// RegisterBuiltInFunctions registers all built in functions for the flow +func (f *FlowExecutor) RegisterBuiltInFunctions() error { + // currently we register following builtin functions + // log -> log to stdout with [JS] prefix should only be used for debugging + // set -> set a variable in template context + // proto(arg ...String) <- this is generic syntax of how a protocol/request binding looks in js + // we only register only those protocols that are available in template + + // we also register a map datatype called template with all template variables + // template -> all template variables are available in js template object + if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value { + // TODO: verify string interpolation and handle multiple args arg := call.Argument(0).Export() gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), arg) return goja.Null() @@ -170,20 +189,10 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) return err } - if err := f.jsVM.Set("poll", func(call goja.FunctionCall) goja.Value { - var m map[string]interface{} = f.options.TemplateCtx.GetAll() - _ = f.jsVM.Set("template", m) - return goja.Null() - }); err != nil { - return err - } - if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value { varName := call.Argument(0).Export() varValue := call.Argument(1).Export() f.options.TemplateCtx.Set(varName.(string), varValue) - // gologger.Debug().Msgf("JS: set %s to %s", varName, varValue) - // fmt.Printf("log: set %s to %s\n", varName, varValue) return goja.Null() }); err != nil { return err @@ -197,7 +206,7 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) } // register all protocols - for name, fn := range VarRegistry { + for name, fn := range f.protoFunctions { if err := f.jsVM.Set(name, fn); err != nil { return err } @@ -208,7 +217,6 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) return err } f.program = program - return nil } @@ -222,9 +230,11 @@ func (f *FlowExecutor) Execute() (bool, error) { return f.results.Load(), nil } +// ReadDataFromFile reads data from file respecting sandbox options func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { values := []string{} - reader, err := f.options.Catalog.OpenFile(payload) + // load file respecting sandbox + reader, err := f.options.Options.LoadHelperFile(payload, f.options.TemplatePath, f.options.Catalog) if err != nil { return values, err } diff --git a/v2/pkg/protocols/common/generators/generators.go b/v2/pkg/protocols/common/generators/generators.go index 98fc6aa138..c17fef84c8 100644 --- a/v2/pkg/protocols/common/generators/generators.go +++ b/v2/pkg/protocols/common/generators/generators.go @@ -6,7 +6,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v2/pkg/catalog" - "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) // PayloadGenerator is the generator struct for generating payloads @@ -14,10 +14,11 @@ type PayloadGenerator struct { Type AttackType catalog catalog.Catalog payloads map[string][]string + options *types.Options } // New creates a new generator structure for payload generation -func New(payloads map[string]interface{}, attackType AttackType, templatePath string, allowLocalFileAccess bool, catalog catalog.Catalog, customAttackType string) (*PayloadGenerator, error) { +func New(payloads map[string]interface{}, attackType AttackType, templatePath string, catalog catalog.Catalog, customAttackType string, opts *types.Options) (*PayloadGenerator, error) { if attackType.String() == "" { attackType = BatteringRamAttack } @@ -38,12 +39,12 @@ func New(payloads map[string]interface{}, attackType AttackType, templatePath st } } - generator := &PayloadGenerator{catalog: catalog} + generator := &PayloadGenerator{catalog: catalog, options: opts} if err := generator.validate(payloadsFinal, templatePath); err != nil { return nil, err } - compiled, err := generator.loadPayloads(payloadsFinal, templatePath, config.DefaultConfig.TemplatesDirectory, allowLocalFileAccess) + compiled, err := generator.loadPayloads(payloadsFinal, templatePath) if err != nil { return nil, err } diff --git a/v2/pkg/protocols/common/generators/generators_test.go b/v2/pkg/protocols/common/generators/generators_test.go index 7187808904..2226188dca 100644 --- a/v2/pkg/protocols/common/generators/generators_test.go +++ b/v2/pkg/protocols/common/generators/generators_test.go @@ -6,13 +6,14 @@ import ( "github.com/stretchr/testify/require" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) func TestBatteringRamGenerator(t *testing.T) { usernames := []string{"admin", "password"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames}, BatteringRamAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -32,7 +33,7 @@ func TestPitchforkGenerator(t *testing.T) { passwords := []string{"password1", "password2", "password3"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, PitchForkAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -54,7 +55,7 @@ func TestClusterbombGenerator(t *testing.T) { passwords := []string{"admin", "password", "token"} catalogInstance := disk.NewCatalog("") - generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", false, catalogInstance, "") + generator, err := New(map[string]interface{}{"username": usernames, "password": passwords}, ClusterBombAttack, "", catalogInstance, "", getOptions(false)) require.Nil(t, err, "could not create generator") iterator := generator.NewIterator() @@ -83,3 +84,9 @@ func TestClusterbombGenerator(t *testing.T) { } require.Equal(t, 3, count, "could not get correct clusterbomb counts") } + +func getOptions(allowLocalFileAccess bool) *types.Options { + opts := types.DefaultOptions() + opts.AllowLocalFileAccess = allowLocalFileAccess + return opts +} diff --git a/v2/pkg/protocols/common/generators/load.go b/v2/pkg/protocols/common/generators/load.go index 92ec932019..390c8141e4 100644 --- a/v2/pkg/protocols/common/generators/load.go +++ b/v2/pkg/protocols/common/generators/load.go @@ -2,7 +2,7 @@ package generators import ( "bufio" - "path/filepath" + "io" "strings" "github.com/pkg/errors" @@ -11,7 +11,7 @@ import ( ) // loadPayloads loads the input payloads from a map to a data map -func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath, templateDirectory string, allowLocalFileAccess bool) (map[string][]string, error) { +func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, templatePath string) (map[string][]string, error) { loadedPayloads := make(map[string][]string) for name, payload := range payloads { @@ -22,18 +22,11 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, if len(elements) >= 2 { loadedPayloads[name] = elements } else { - if !allowLocalFileAccess { - pt = filepath.Clean(pt) - templateAbsPath, err := filepath.Abs(templatePath) - if err != nil { - return nil, errors.Wrap(err, "could not get absolute path") - } - templatePathDir := filepath.Dir(templateAbsPath) - if !(templatePathDir != "/" && strings.HasPrefix(pt, templatePathDir)) && !strings.HasPrefix(pt, templateDirectory) { - return nil, errors.New("denied payload file path specified") - } + file, err := generator.options.LoadHelperFile(pt, templatePath, generator.catalog) + if err != nil { + return nil, errors.Wrap(err, "could not load payload file") } - payloads, err := generator.loadPayloadsFromFile(pt) + payloads, err := generator.loadPayloadsFromFile(file) if err != nil { return nil, errors.Wrap(err, "could not load payloads") } @@ -47,13 +40,8 @@ func (generator *PayloadGenerator) loadPayloads(payloads map[string]interface{}, } // loadPayloadsFromFile loads a file to a string slice -func (generator *PayloadGenerator) loadPayloadsFromFile(filepath string) ([]string, error) { +func (generator *PayloadGenerator) loadPayloadsFromFile(file io.ReadCloser) ([]string, error) { var lines []string - - file, err := generator.catalog.OpenFile(filepath) - if err != nil { - return nil, err - } defer file.Close() scanner := bufio.NewScanner(file) diff --git a/v2/pkg/protocols/common/generators/load_test.go b/v2/pkg/protocols/common/generators/load_test.go index 28803b097b..49c00ef2f9 100644 --- a/v2/pkg/protocols/common/generators/load_test.go +++ b/v2/pkg/protocols/common/generators/load_test.go @@ -2,66 +2,119 @@ package generators import ( "os" + "os/exec" "path/filepath" "testing" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" osutils "github.com/projectdiscovery/utils/os" "github.com/stretchr/testify/require" ) func TestLoadPayloads(t *testing.T) { - tempdir, err := os.MkdirTemp("", "templates-*") - require.NoError(t, err, "could not create temp dir") - defer os.RemoveAll(tempdir) + // since we are changing value of global variable i.e templates directory + // run this test as subprocess + if os.Getenv("LOAD_PAYLOAD_NO_ACCESS") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestLoadPayloadsWithAccess") + cmd.Env = append(os.Environ(), "LOAD_PAYLOAD_NO_ACCESS=1") + err := cmd.Run() + if e, ok := err.(*exec.ExitError); ok && !e.Success() { + return + } + if err != nil { + t.Fatalf("process ran with err %v, want exit status 1", err) + } + } + templateDir := getTemplatesDir(t) + config.DefaultConfig.SetTemplatesDir(templateDir) - generator := &PayloadGenerator{catalog: disk.NewCatalog(tempdir)} - - fullpath := filepath.Join(tempdir, "payloads.txt") - err = os.WriteFile(fullpath, []byte("test\nanother"), 0777) - require.NoError(t, err, "could not write payload") + generator := &PayloadGenerator{catalog: disk.NewCatalog(templateDir), options: getOptions(false)} + fullpath := filepath.Join(templateDir, "payloads.txt") // Test sandbox t.Run("templates-directory", func(t *testing.T) { + // testcase when loading file from template directory and template file is in root + // expected to succeed values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/test", tempdir, false) + }, "/test") require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) t.Run("templates-path-relative", func(t *testing.T) { + // testcase when loading file from template directory and template file is current working directory + // expected to fail since this is LFI _, err := generator.loadPayloads(map[string]interface{}{ "new": "../../../../../../../../../etc/passwd", - }, ".", tempdir, false) + }, ".") require.Error(t, err, "could load payloads") }) t.Run("template-directory", func(t *testing.T) { + // testcase when loading file from template directory and template file is inside template directory + // expected to succeed values, err := generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, filepath.Join(tempdir, "test.yaml"), "/test", false) + }, filepath.Join(templateDir, "test.yaml")) require.NoError(t, err, "could not load payloads") require.Equal(t, map[string][]string{"new": {"test", "another"}}, values, "could not get values") }) - t.Run("no-sandbox-unix", func(t *testing.T) { - if osutils.IsWindows() { - return - } - _, err := generator.loadPayloads(map[string]interface{}{ - "new": "/etc/passwd", - }, "/random", "/test", true) - require.NoError(t, err, "could load payloads") - }) + t.Run("invalid", func(t *testing.T) { + // testcase when loading file from /etc/passwd and template file is at root i.e / + // expected to fail since this is LFI values, err := generator.loadPayloads(map[string]interface{}{ "new": "/etc/passwd", - }, "/random", "/test", false) + }, "/random") require.Error(t, err, "could load payloads") require.Equal(t, 0, len(values), "could get values") + // testcase when loading file from template directory and template file is at root i.e / + // expected to succeed values, err = generator.loadPayloads(map[string]interface{}{ "new": fullpath, - }, "/random", "/test", false) - require.Error(t, err, "could load payloads") - require.Equal(t, 0, len(values), "could get values") + }, "/random") + require.NoError(t, err, "could load payloads %v", values) + require.Equal(t, 1, len(values), "could get values") + require.Equal(t, []string{"test", "another"}, values["new"], "could get values") + }) +} + +func TestLoadPayloadsWithAccess(t *testing.T) { + // since we are changing value of global variable i.e templates directory + // run this test as subprocess + if os.Getenv("LOAD_PAYLOAD_WITH_ACCESS") != "1" { + cmd := exec.Command(os.Args[0], "-test.run=TestLoadPayloadsWithAccess") + cmd.Env = append(os.Environ(), "LOAD_PAYLOAD_WITH_ACCESS=1") + err := cmd.Run() + if e, ok := err.(*exec.ExitError); ok && !e.Success() { + return + } + if err != nil { + t.Fatalf("process ran with err %v, want exit status 1", err) + } + } + templateDir := getTemplatesDir(t) + config.DefaultConfig.SetTemplatesDir(templateDir) + + generator := &PayloadGenerator{catalog: disk.NewCatalog(templateDir), options: getOptions(true)} + + t.Run("no-sandbox-unix", func(t *testing.T) { + if osutils.IsWindows() { + return + } + _, err := generator.loadPayloads(map[string]interface{}{ + "new": "/etc/passwd", + }, "/random") + require.NoError(t, err, "could load payloads") }) } + +func getTemplatesDir(t *testing.T) string { + tempdir, err := os.MkdirTemp("", "templates-*") + require.NoError(t, err, "could not create temp dir") + fullpath := filepath.Join(tempdir, "payloads.txt") + err = os.WriteFile(fullpath, []byte("test\nanother"), 0777) + require.NoError(t, err, "could not write payload") + return tempdir +} diff --git a/v2/pkg/protocols/dns/dns.go b/v2/pkg/protocols/dns/dns.go index 1cc8302e57..232427058a 100644 --- a/v2/pkg/protocols/dns/dns.go +++ b/v2/pkg/protocols/dns/dns.go @@ -172,7 +172,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 0d1d09e5e7..6876c70ac8 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -106,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { if len(request.Payloads) > 0 { var err error - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, options.TemplatePath, options.Catalog, options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index c14c6fe3cb..18595b0cbb 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -72,8 +72,8 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, // 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path // add template context values to dynamicValues (this takes care of self-contained and other types of requests) - // dynamicValues should be given preference over templateCtx values - dynamicValues = generators.MergeMaps(r.request.options.TemplateCtx.GetAll(), dynamicValues) + // Note: `iterate-all` and flow are mutually exclusive. flow uses templateCtx and iterate-all uses dynamicValues + dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.TemplateCtx.GetAll()) if r.request.SelfContained { return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues) } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 552d9229bd..7cef0a2e18 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -353,7 +353,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 9629860bfc..7e5f40040d 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -426,7 +426,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa var gotErr error var skip bool if len(gotDynamicValues) > 0 { - // fmt.Printf("dynamic values %v and templateCtx values %v\n", gotDynamicValues, request.options.TemplateCtx.GetAll()) operators.MakeDynamicValuesCallback(gotDynamicValues, request.IterateAll, func(data map[string]interface{}) bool { if skip, gotErr = executeFunc(inputData, payloads, data); skip || gotErr != nil { return true diff --git a/v2/pkg/protocols/http/request_generator_test.go b/v2/pkg/protocols/http/request_generator_test.go index 89447b9936..c57291d584 100644 --- a/v2/pkg/protocols/http/request_generator_test.go +++ b/v2/pkg/protocols/http/request_generator_test.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/types" ) func TestRequestGeneratorPaths(t *testing.T) { @@ -34,7 +35,7 @@ func TestRequestGeneratorClusterBombSingle(t *testing.T) { Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`}, } catalogInstance := disk.NewCatalog("") - req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "") + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance, "", types.DefaultOptions()) require.Nil(t, err, "could not create generator") generator := req.newGenerator(false) @@ -58,7 +59,7 @@ func TestRequestGeneratorClusterBombMultipleRaw(t *testing.T) { Raw: []string{`GET /{{username}}:{{password}} HTTP/1.1`, `GET /{{username}}@{{password}} HTTP/1.1`}, } catalogInstance := disk.NewCatalog("") - req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", false, catalogInstance, "") + req.generator, err = generators.New(req.Payloads, req.AttackType.Value, "", catalogInstance, "", types.DefaultOptions()) require.Nil(t, err, "could not create generator") generator := req.newGenerator(false) diff --git a/v2/pkg/protocols/network/network.go b/v2/pkg/protocols/network/network.go index c344538f58..7a5795ac97 100644 --- a/v2/pkg/protocols/network/network.go +++ b/v2/pkg/protocols/network/network.go @@ -184,7 +184,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, request.options.Catalog, request.options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Catalog, request.options.Options.AttackType, request.options.Options) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 10c4f27692..d8b3488615 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -98,7 +98,10 @@ type ExecutorOptions struct { // AddTemplateVars adds vars to template context with given template type as prefix // this method is no-op if template is not multi protocol func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, vars map[string]interface{}) { - if e.ProtocolType != templateTypes.MultiProtocol || e.Flow != "" { + // if we wan't to disable adding response variables and other variables to template context + // this is the statement that does it . template context is currently only enabled for + // multiprotocol and flow templates + if e.ProtocolType != templateTypes.MultiProtocol && e.Flow == "" { // no-op if not multi protocol template return } @@ -115,7 +118,7 @@ func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolTyp // AddTemplateVar adds given var to template context with given template type as prefix // this method is no-op if template is not multi protocol func (e *ExecutorOptions) AddTemplateVar(prefix, key string, value interface{}) { - if e.ProtocolType != templateTypes.MultiProtocol { + if e.ProtocolType != templateTypes.MultiProtocol && e.Flow == "" { // no-op if not multi protocol template return } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index a2d811eef1..3f90d21109 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -106,7 +106,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.dialer = client if len(request.Payloads) > 0 { - request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, request.options.Options.AllowLocalFileAccess, options.Catalog, options.Options.AttackType) + request.generator, err = generators.New(request.Payloads, request.AttackType.Value, request.options.TemplatePath, options.Catalog, options.Options.AttackType, types.DefaultOptions()) if err != nil { return errors.Wrap(err, "could not parse payloads") } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 756f98f200..99dfae52dc 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -3,7 +3,6 @@ package templates import ( "fmt" "io" - "os" "path/filepath" "reflect" "strings" @@ -20,6 +19,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/retryablehttp-go" + errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) @@ -241,13 +241,20 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option options.Variables = template.Variables } - // TODO: we should add a syntax check here + // TODO: we should add a syntax check here or somehow use a javascript linter + // simplest option for now seems to compile using goja and see if it fails if strings.TrimSpace(template.Flow) != "" { if len(template.Flow) > 0 && filepath.Ext(template.Flow) == ".js" && fileutil.FileExists(template.Flow) { - // TODO: this is sandbox bypass, we should remove it - // move sandbox check to config so it can be reused here and in the generator - if bin, err := os.ReadFile(template.Flow); err == nil { + // load file respecting sandbox + file, err := options.Options.LoadHelperFile(template.Flow, options.TemplatePath, options.Catalog) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("loading flow file from %v denied", template.Flow) + } + defer file.Close() + if bin, err := io.ReadAll(file); err == nil { template.Flow = string(bin) + } else { + return nil, errorutil.NewWithErr(err).Msgf("something went wrong failed to read file") } } options.Flow = template.Flow diff --git a/v2/pkg/types/types.go b/v2/pkg/types/types.go index 9aef497cf3..e6f88bc0eb 100644 --- a/v2/pkg/types/types.go +++ b/v2/pkg/types/types.go @@ -2,12 +2,17 @@ package types import ( "io" + "os" + "path/filepath" "strings" "time" "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v2/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" ) @@ -464,3 +469,29 @@ func (options *Options) ParseHeadlessOptionalArguments() map[string]string { } return optionalArguments } + +// LoadHelperFile loads a helper file needed for the template +// this respects the sandbox rules and only loads files from +// allowed directories +func (options *Options) LoadHelperFile(filePath, templatePath string, catalog catalog.Catalog) (io.ReadCloser, error) { + if !options.AllowLocalFileAccess { + filePath = filepath.Clean(filePath) + templateAbsPath, err := filepath.Abs(templatePath) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not get absolute path") + } + templateDirectory := config.DefaultConfig.TemplatesDirectory + templatePathDir := filepath.Dir(templateAbsPath) + if !(templatePathDir != "/" && strings.HasPrefix(filePath, templatePathDir)) && !strings.HasPrefix(filePath, templateDirectory) { + return nil, errorutil.New("denied payload file path specified") + } + } + if catalog != nil { + return catalog.OpenFile(filePath) + } + f, err := os.Open(filePath) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not open file %v", filePath) + } + return f, nil +} From 41f13bcb66151680d355fa5cbac26a539b455eff Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 7 Aug 2023 22:03:55 +0530 Subject: [PATCH 09/38] fix failing integration tests --- integration_tests/protocols/code/py-env-var.yaml | 2 +- integration_tests/protocols/code/py-file.yaml | 2 +- integration_tests/protocols/code/py-interactsh.yaml | 2 +- integration_tests/protocols/code/py-snippet.yaml | 2 +- v2/pkg/protocols/common/executer/executer.go | 10 +++------- v2/pkg/protocols/headless/headless.go | 3 ++- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 2e3a3a6194..9a7b0b3865 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 490a00463044022013021390d00638aa2c178f386cd45a9bbd560ec19ed834755d1e2379b967dcdd0220535a7763e78a16fa149a8a04ebeb25d9124a046e39810d828ba37a1ea1109066 \ No newline at end of file +# digest: 490a0046304402205f67e50cb11230efd286c7cef252f09565162375d86298e166d40ca860ca81ff022079652101b5e36405b632f0905bc0350b1d114bcd6b6da47345e454d8fff3ea2d \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 7af0b3f214..daa3491956 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 490a0046304402203269b5ac13def28e907e7529863b92c1fb4b8e20328c284a4ed99bf5d871505f02200fb8b8ed0b6a8dc00dc999d9acc9292681a34166c3c760411ac764fd343d0c0a \ No newline at end of file +# digest: 4b0a00483046022100bbc391f379fc1745e56f631d83d74411df6f575ea6213e785069e9fd6f87f41b022100bce8125ba6223aa8fde7703708f02c6b5015cf43043947bb77abca9e1906b524 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index af4dfba02f..0686b59644 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 490a004630440220740d6c3ac20692f051b7642c7ffed00ff4b804d60110b067a7083e56bdcae17502202a4d7256e9a10a717556ff44de55dceeb1521afe9cda8955ea80dc401495c259 \ No newline at end of file +# digest: 490a00463044022009697d61506d5937791f83dc6407ea023238832593d953ffa53885c105708030022079d88c4bf6d2ebdf241d16c1d16c2b521c8a6332743cdbca4144a9a32b54c6df \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 51aa8d19be..5611467f08 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a00473045022100bfc50dec85d30f8ce57bf2a2ca22d9759e1045426781bdbced085bd42c4db9db02201cc3510758ce06c50976910919a71a2310564ad1815bffd55e0864335a4735f0 \ No newline at end of file +# digest: 4b0a00483046022100e2e8866788f2f225663514736ef03fa7575f97e6a6a6c5963a98c1af49f422c8022100c1a50ae5cdf15cacc4c72bdac72b7101f1cb7fe0ace97fb4f8148de99db6372d \ No newline at end of file diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 4b5d2b250b..ad00c3a954 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -81,15 +81,11 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { // in that case we can skip it, otherwise we've to show failure in // case of matcher-status flag. if !event.HasOperatorResult() && !event.UsesInteractsh { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } lastMatcherEvent = event } else { - if ok := writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient); !ok { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } + if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { + results.CompareAndSwap(false, true) + } else { lastMatcherEvent = event } } diff --git a/v2/pkg/protocols/headless/headless.go b/v2/pkg/protocols/headless/headless.go index 6876c70ac8..32dc1af8f0 100644 --- a/v2/pkg/protocols/headless/headless.go +++ b/v2/pkg/protocols/headless/headless.go @@ -91,6 +91,8 @@ func (request *Request) GetID() string { // Compile compiles the protocol request for further execution. func (request *Request) Compile(options *protocols.ExecutorOptions) error { + request.options = options + // TODO: logic similar to network + http => probably can be refactored // Resolve payload paths from vars if they exists for name, payload := range options.Options.Vars.AsMap() { @@ -136,7 +138,6 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } request.CompiledOperators = compiled } - request.options = options if len(request.Fuzzing) > 0 { for _, rule := range request.Fuzzing { From 3677ff203307ea25f98832e0a348fb155bd996d6 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 8 Aug 2023 00:14:57 +0530 Subject: [PATCH 10/38] JS: log: print in vardump format --- .../common/executer/flow_executor.go | 12 +++- .../common/executer/flow_executor_test.go | 59 +++++++++++++++++++ .../testcases/nuclei-flow-dns-id.yaml | 42 +++++++++++++ .../executer/testcases/nuclei-flow-dns.yaml | 41 +++++++++++++ 4 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 v2/pkg/protocols/common/executer/flow_executor_test.go create mode 100644 v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-id.yaml create mode 100644 v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index ff7209c0f2..a0f2802976 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -13,6 +13,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps" @@ -66,6 +67,8 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) if str, ok := v.(string); ok && len(str) < 150 && fileutil.FileExists(str) { if value, err := f.ReadDataFromFile(str); err == nil { allVars[k] = value + } else { + gologger.Warning().Msgf("could not load file '%s' for variable '%s': %s", str, k, err) } } } @@ -183,7 +186,14 @@ func (f *FlowExecutor) RegisterBuiltInFunctions() error { if err := f.jsVM.Set("log", func(call goja.FunctionCall) goja.Value { // TODO: verify string interpolation and handle multiple args arg := call.Argument(0).Export() - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), arg) + switch value := arg.(type) { + case string: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + case map[string]interface{}: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) + default: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + } return goja.Null() }); err != nil { return err diff --git a/v2/pkg/protocols/common/executer/flow_executor_test.go b/v2/pkg/protocols/common/executer/flow_executor_test.go new file mode 100644 index 0000000000..f94cabf56f --- /dev/null +++ b/v2/pkg/protocols/common/executer/flow_executor_test.go @@ -0,0 +1,59 @@ +package executer_test + +import ( + "context" + "log" + "testing" + "time" + + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v2/pkg/catalog/disk" + "github.com/projectdiscovery/nuclei/v2/pkg/parsers" + "github.com/projectdiscovery/nuclei/v2/pkg/progress" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/templates" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" + "github.com/projectdiscovery/ratelimit" + "github.com/stretchr/testify/require" +) + +var executerOpts protocols.ExecutorOptions + +func setup() { + options := testutils.DefaultOptions + testutils.Init(options) + progressImpl, _ := progress.NewStatsTicker(0, false, false, false, false, 0) + + executerOpts = protocols.ExecutorOptions{ + Output: testutils.NewMockOutputWriter(), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + } + workflowLoader, err := parsers.NewLoader(&executerOpts) + if err != nil { + log.Fatalf("Could not create workflow loader: %s\n", err) + } + executerOpts.WorkflowLoader = workflowLoader +} + +func TestFlowTemplateWithID(t *testing.T) { + // test + setup() + Template, err := templates.Parse("testcases/nuclei-flow-dns.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("hackerone.com")) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) +} diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-id.yaml b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-id.yaml new file mode 100644 index 0000000000..d13121fa0b --- /dev/null +++ b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-id.yaml @@ -0,0 +1,42 @@ +id: nuclei-flow-dns + +info: + name: Nuclei flow dns + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("fetch-ns"); + template["nameservers"].forEach(nameserver => { + set("nameserver",nameserver); + dns("probe-ns"); + }); + +dns: + - id: "fetch-ns" + name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - id: "probe-ns" + name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml new file mode 100644 index 0000000000..4bf690cb51 --- /dev/null +++ b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml @@ -0,0 +1,41 @@ +id: nuclei-flow-dns + +info: + name: Nuclei flow dns + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("0"); + log(template) + template["nameservers"].forEach(nameserver => { + set("nameserver",nameserver); + dns("1"); + }); + +dns: + - name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file From b7cc2c359e7d497c6b5218504d3cd5511c785387 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 8 Aug 2023 00:40:15 +0530 Subject: [PATCH 11/38] fix missing id in protocols --- v2/pkg/protocols/common/executer/flow_executor.go | 4 +++- v2/pkg/protocols/common/executer/flow_executor_test.go | 6 ++++++ .../common/executer/testcases/nuclei-flow-dns.yaml | 2 +- v2/pkg/protocols/ssl/ssl.go | 3 +++ v2/pkg/protocols/websocket/websocket.go | 2 ++ v2/pkg/protocols/whois/whois.go | 3 +++ v2/pkg/templates/compile.go | 2 ++ 7 files changed, 20 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index a0f2802976..493a9b222d 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -43,7 +43,9 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) f.results = new(atomic.Bool) } // store all dynamic variables and other variables here - f.options.TemplateCtx = contextargs.New() + if f.options.TemplateCtx == nil { + f.options.TemplateCtx = contextargs.New() + } // create a new js vm/runtime f.jsVM = goja.New() diff --git a/v2/pkg/protocols/common/executer/flow_executor_test.go b/v2/pkg/protocols/common/executer/flow_executor_test.go index f94cabf56f..b4497cae70 100644 --- a/v2/pkg/protocols/common/executer/flow_executor_test.go +++ b/v2/pkg/protocols/common/executer/flow_executor_test.go @@ -56,4 +56,10 @@ func TestFlowTemplateWithID(t *testing.T) { gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("hackerone.com")) require.Nil(t, err, "could not execute template") require.True(t, gotresults) + + value, ok := Template.Options.TemplateCtx.Get("nameservers") + require.True(t, ok) + if value != nil { + require.True(t, len(value.([]string)) > 0) + } } diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml index 4bf690cb51..6a46c1e8c9 100644 --- a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml +++ b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml @@ -9,10 +9,10 @@ info: flow: | dns("0"); - log(template) template["nameservers"].forEach(nameserver => { set("nameserver",nameserver); dns("1"); + log(template) }); dns: diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index b13c3030cd..6817028840 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -41,6 +41,9 @@ type Request struct { operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-" json:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` + // description: | // Address contains address for the request Address string `yaml:"address,omitempty" json:"address,omitempty" jsonschema:"title=address for the ssl request,description=Address contains address for the request"` diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index 3f90d21109..b5631d2144 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -42,6 +42,8 @@ type Request struct { operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-" json:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` // description: | // Address contains address for the request Address string `yaml:"address,omitempty" json:"address,omitempty" jsonschema:"title=address for the websocket request,description=Address contains address for the request"` diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index 5d538ee095..11e68224b1 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -34,6 +34,9 @@ type Request struct { operators.Operators `yaml:",inline,omitempty" json:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-" json:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the request,description=ID of the network request"` + // description: | // Query contains query for the request Query string `yaml:"query,omitempty" json:"query,omitempty" jsonschema:"title=query for the WHOIS request,description=Query contains query for the request"` diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 99dfae52dc..67ad8288bf 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -264,6 +264,8 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option options.TemplateCtx = contextargs.New() options.ProtocolType = template.Type() options.Constants = template.Constants + + template.Options = &options // If no requests, and it is also not a workflow, return error. if template.Requests() == 0 { return nil, fmt.Errorf("no requests defined for %s", template.ID) From 4fffe106011ae11d0ea4ab36a23f2eb8bb26de19 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 8 Aug 2023 01:29:10 +0530 Subject: [PATCH 12/38] fix proto prefix in template context --- v2/pkg/protocols/code/code.go | 4 +- .../common/executer/flow_executor_test.go | 42 +++++++++- .../testcases/nuclei-flow-dns-prefix.yaml | 41 ++++++++++ .../executer/testcases/nuclei-flow-dns.yaml | 2 +- v2/pkg/protocols/dns/request.go | 2 +- v2/pkg/protocols/headless/request.go | 2 +- v2/pkg/protocols/http/request.go | 2 +- v2/pkg/protocols/network/request.go | 2 +- v2/pkg/protocols/offlinehttp/request.go | 2 +- v2/pkg/protocols/protocols.go | 18 +++-- v2/pkg/protocols/ssl/ssl.go | 4 +- v2/pkg/protocols/websocket/websocket.go | 2 +- v2/pkg/protocols/whois/whois.go | 2 +- v2/pkg/templates/compile.go | 5 ++ v2/pkg/templates/templates.go | 76 +++++++++++++++++++ 15 files changed, 187 insertions(+), 19 deletions(-) create mode 100644 v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml diff --git a/v2/pkg/protocols/code/code.go b/v2/pkg/protocols/code/code.go index d601125466..e7f710074b 100644 --- a/v2/pkg/protocols/code/code.go +++ b/v2/pkg/protocols/code/code.go @@ -32,6 +32,8 @@ type Request struct { operators.Operators `yaml:",inline,omitempty"` CompiledOperators *operators.Operators `yaml:"-"` + // ID is the optional id of the request + ID string `yaml:"id,omitempty" json:"id,omitempty" jsonschema:"title=id of the dns request,description=ID is the optional ID of the DNS Request"` // description: | // Engine type Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine,enum=python,enum=powershell,enum=command"` @@ -96,7 +98,7 @@ func (request *Request) Requests() int { // GetID returns the ID for the request if any. func (request *Request) GetID() string { - return "" + return request.ID } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. diff --git a/v2/pkg/protocols/common/executer/flow_executor_test.go b/v2/pkg/protocols/common/executer/flow_executor_test.go index b4497cae70..66089daca4 100644 --- a/v2/pkg/protocols/common/executer/flow_executor_test.go +++ b/v2/pkg/protocols/common/executer/flow_executor_test.go @@ -42,7 +42,7 @@ func setup() { executerOpts.WorkflowLoader = workflowLoader } -func TestFlowTemplateWithID(t *testing.T) { +func TestFlowTemplateWithIndex(t *testing.T) { // test setup() Template, err := templates.Parse("testcases/nuclei-flow-dns.yaml", nil, executerOpts) @@ -63,3 +63,43 @@ func TestFlowTemplateWithID(t *testing.T) { require.True(t, len(value.([]string)) > 0) } } + +func TestFlowTemplateWithID(t *testing.T) { + setup() + Template, err := templates.Parse("testcases/nuclei-flow-dns-id.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("hackerone.com")) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) + + value, ok := Template.Options.TemplateCtx.Get("nameservers") + require.True(t, ok) + if value != nil { + require.True(t, len(value.([]string)) > 0) + } +} + +func TestFlowWithProtoPrefix(t *testing.T) { + // test + setup() + Template, err := templates.Parse("testcases/nuclei-flow-dns.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("hackerone.com")) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) + + // while there are lot of variables lets just look for only these + // protoVars := []string{"dns_"} +} diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml new file mode 100644 index 0000000000..a054b4c4fa --- /dev/null +++ b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml @@ -0,0 +1,41 @@ +id: nuclei-flow-dns + +info: + name: Nuclei flow dns + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("fetch-ns"); + template["nameservers"].forEach(nameserver => { + set("nameserver",nameserver); + dns("1"); + }); + +dns: + - id: "fetch-ns" + name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml index 6a46c1e8c9..e26e37013e 100644 --- a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml +++ b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml @@ -12,8 +12,8 @@ flow: | template["nameservers"].forEach(nameserver => { set("nameserver",nameserver); dns("1"); - log(template) }); + log(template) dns: - name: "{{FQDN}}" diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 8d05d37172..845ac9a816 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -151,7 +151,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData) // expose response variables in proto_var format // this is no-op if the template is not a multi protocol template - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(request.Type(), request.ID, outputEvent) for k, v := range previous { outputEvent[k] = v } diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index 049cfb5a53..eb0e18c004 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -156,7 +156,7 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory()) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(request.Type(), request.ID, outputEvent) outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) for k, v := range out { outputEvent[k] = v diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 7e5f40040d..88db9387e2 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -721,7 +721,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(request.Type(), request.ID, outputEvent) outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index 744f4fd6dc..af7538c957 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -281,7 +281,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac response := responseBuilder.String() outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(request.Type(), request.ID, outputEvent) outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) outputEvent["ip"] = request.dialer.GetDialedIP(hostname) if request.options.StopAtFirstMatch { diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 106023e9ee..58386c3a32 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -88,7 +88,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), utils.HeadersToString(resp.Header), 0, nil) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(request.Type(), request.GetID(), outputEvent) outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) outputEvent["ip"] = "" for k, v := range previous { diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index d8b3488615..0d65e36ded 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -97,17 +97,19 @@ type ExecutorOptions struct { // AddTemplateVars adds vars to template context with given template type as prefix // this method is no-op if template is not multi protocol -func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, vars map[string]interface{}) { +func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, reqID string, vars map[string]interface{}) { // if we wan't to disable adding response variables and other variables to template context // this is the statement that does it . template context is currently only enabled for // multiprotocol and flow templates if e.ProtocolType != templateTypes.MultiProtocol && e.Flow == "" { - // no-op if not multi protocol template + // no-op if not multi protocol template or flow template return } for k, v := range vars { if !stringsutil.EqualFoldAny(k, "template-id", "template-info", "template-path") { - if templateType < templateTypes.InvalidProtocol { + if reqID != "" { + k = reqID + "_" + k + } else if templateType < templateTypes.InvalidProtocol { k = templateType.String() + "_" + k } e.TemplateCtx.Set(k, v) @@ -117,13 +119,15 @@ func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolTyp // AddTemplateVar adds given var to template context with given template type as prefix // this method is no-op if template is not multi protocol -func (e *ExecutorOptions) AddTemplateVar(prefix, key string, value interface{}) { +func (e *ExecutorOptions) AddTemplateVar(templateType templateTypes.ProtocolType, reqID string, key string, value interface{}) { if e.ProtocolType != templateTypes.MultiProtocol && e.Flow == "" { - // no-op if not multi protocol template + // no-op if not multi protocol template or flow template return } - if prefix != "" { - key = prefix + "_" + key + if reqID != "" { + key = reqID + "_" + key + } else if templateType < templateTypes.InvalidProtocol { + key = templateType.String() + "_" + key } e.TemplateCtx.Set(key, value) } diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 6817028840..f12f39eaf6 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -270,7 +270,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if tag == "" || f.IsZero() { continue } - request.options.AddTemplateVar(request.Type().String(), tag, f.Value()) + request.options.AddTemplateVar(request.Type(), request.ID, tag, f.Value()) data[tag] = f.Value() } @@ -289,7 +289,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if tag == "" || f.IsZero() { continue } - request.options.AddTemplateVar(request.Type().String(), tag, f.Value()) + request.options.AddTemplateVar(request.Type(), request.ID, tag, f.Value()) data[tag] = f.Value() } diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index b5631d2144..e65cad4498 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -267,7 +267,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam data["ip"] = request.dialer.GetDialedIP(hostname) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), data) + request.options.AddTemplateVars(request.Type(), request.ID, data) data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) for k, v := range previous { diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index 11e68224b1..942eb3ba53 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -137,7 +137,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa data["response"] = jsonDataString // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), data) + request.options.AddTemplateVars(request.Type(), request.ID, data) data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 67ad8288bf..ab77fe164a 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -241,6 +241,11 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option options.Variables = template.Variables } + // if more than 1 request per protocol exist we add request id to protocol request + // since in template context we have proto_prefix for each protocol it is overwritten + // if request id is not present + template.validateAllRequestIDs() + // TODO: we should add a syntax check here or somehow use a javascript linter // simplest option for now seems to compile using goja and see if it fails if strings.TrimSpace(template.Flow) != "" { diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 600fe352d7..7a705fb97b 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -3,6 +3,7 @@ package templates import ( "encoding/json" + "strconv" validate "github.com/go-playground/validator/v10" "github.com/projectdiscovery/nuclei/v2/pkg/model" @@ -180,9 +181,84 @@ func (template *Template) Type() types.ProtocolType { } } +// validateAllRequestIDs check if that protocol already has given id if not +// then is is manually set to proto_index +func (template *Template) validateAllRequestIDs() { + // this is required in multiprotocol and flow where we save response variables + // and all other data in template context if template as two requests in a protocol + // then it is overwritten to avoid this we use proto_index as request ID + if len(template.RequestsCode) > 1 { + for i, req := range template.RequestsCode { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsDNS) > 1 { + for i, req := range template.RequestsDNS { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsFile) > 1 { + for i, req := range template.RequestsFile { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsHTTP) > 1 { + for i, req := range template.RequestsHTTP { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsHeadless) > 1 { + for i, req := range template.RequestsHeadless { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + + } + if len(template.RequestsNetwork) > 1 { + for i, req := range template.RequestsNetwork { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsSSL) > 1 { + for i, req := range template.RequestsSSL { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsWebsocket) > 1 { + for i, req := range template.RequestsWebsocket { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + } + if len(template.RequestsWHOIS) > 1 { + for i, req := range template.RequestsWHOIS { + if req.ID == "" { + req.ID = template.Type().String() + "_" + strconv.Itoa(i) + } + } + } +} + // MarshalYAML forces recursive struct validation during marshal operation func (template *Template) MarshalYAML() ([]byte, error) { out, marshalErr := yaml.Marshal(template) + // Review: we are adding requestIDs for templateContext + // if we are using this method then we might need to purge manually added IDS that start with `templatetype_` + // this is only applicable if there are more than 1 request fields in protocol errValidate := validate.New().Struct(template) return out, multierr.Append(marshalErr, errValidate) } From bc4ec8d7da058b37b461ff10d9f6b1f990216101 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 8 Aug 2023 01:49:23 +0530 Subject: [PATCH 13/38] flow: add unit tests --- .../common/executer/flow_executor_test.go | 20 +++++++++++++++++-- .../testcases/nuclei-flow-dns-prefix.yaml | 10 +++++----- .../executer/testcases/nuclei-flow-dns.yaml | 3 +-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/v2/pkg/protocols/common/executer/flow_executor_test.go b/v2/pkg/protocols/common/executer/flow_executor_test.go index 66089daca4..6ecf1d8393 100644 --- a/v2/pkg/protocols/common/executer/flow_executor_test.go +++ b/v2/pkg/protocols/common/executer/flow_executor_test.go @@ -57,6 +57,7 @@ func TestFlowTemplateWithIndex(t *testing.T) { require.Nil(t, err, "could not execute template") require.True(t, gotresults) + // apart from parse->compile->execution this testcase checks if dynamic extracted variables are available value, ok := Template.Options.TemplateCtx.Get("nameservers") require.True(t, ok) if value != nil { @@ -66,6 +67,8 @@ func TestFlowTemplateWithIndex(t *testing.T) { func TestFlowTemplateWithID(t *testing.T) { setup() + // apart from parse->compile->execution this testcase checks support for use custom id for protocol request and invocation of + // the same in js Template, err := templates.Parse("testcases/nuclei-flow-dns-id.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") @@ -88,7 +91,11 @@ func TestFlowTemplateWithID(t *testing.T) { func TestFlowWithProtoPrefix(t *testing.T) { // test setup() - Template, err := templates.Parse("testcases/nuclei-flow-dns.yaml", nil, executerOpts) + + // apart from parse->compile->execution this testcase checks + // mix of custom protocol request id and index is supported in js + // and also validates availability of protocol response variables in template context + Template, err := templates.Parse("testcases/nuclei-flow-dns-prefix.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not @@ -101,5 +108,14 @@ func TestFlowWithProtoPrefix(t *testing.T) { require.True(t, gotresults) // while there are lot of variables lets just look for only these - // protoVars := []string{"dns_"} + protoVars := []string{"dns_0_host", "dns_0_matched", "dns_0_answer", "dns_0_raw", + "probe-ns_host", "probe-ns_matched", "probe-ns_answer", "probe-ns_raw"} + + for _, v := range protoVars { + value, ok := Template.Options.TemplateCtx.Get(v) + require.Truef(t, ok, "could not find variable %s", v) + if value != nil { + require.Truef(t, len(value.(string)) > 0, "variable %s is empty", v) + } + } } diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml index a054b4c4fa..6c040ff7b7 100644 --- a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml +++ b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml @@ -8,15 +8,14 @@ info: reference: https://example-reference-link flow: | - dns("fetch-ns"); + dns("0"); template["nameservers"].forEach(nameserver => { set("nameserver",nameserver); - dns("1"); + dns("probe-ns"); }); dns: - - id: "fetch-ns" - name: "{{FQDN}}" + - name: "{{FQDN}}" type: NS matchers: - type: word @@ -30,7 +29,8 @@ dns: regex: - "IN\tNS\t(.+)" - - name: "{{nameserver}}" + - id: "probe-ns" + name: "{{nameserver}}" type: A class: inet retries: 3 diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml index e26e37013e..5be54ef27d 100644 --- a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml +++ b/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml @@ -13,8 +13,7 @@ flow: | set("nameserver",nameserver); dns("1"); }); - log(template) - + dns: - name: "{{FQDN}}" type: NS From 816d4cc69e9b9c89a4c8616db8e289478389363b Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 8 Aug 2023 04:02:21 +0530 Subject: [PATCH 14/38] conditional flow support using flow --- v2/pkg/protocols/common/executer/executer.go | 3 +- .../common/executer/flow_executor.go | 45 +++++++++++++------ .../executer/testcases/condition-flow.yaml | 28 ++++++++++++ v2/pkg/templates/compile.go | 13 ++++-- 4 files changed, 71 insertions(+), 18 deletions(-) create mode 100644 v2/pkg/protocols/common/executer/testcases/condition-flow.yaml diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index ad00c3a954..6b61833b49 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -183,7 +183,6 @@ func (e *Executer) executeFlow(input *contextargs.Context, callback protocols.Ou if req.Type() == types.MultiProtocol { // multiprotocol execution is also mutually exclusive with flow and is slightly advanced version of executeWithCallbac() // if request type is multiprotocol , then array does not contain any other request type - gologger.Info().Msgf("reqtype is %v", req.Type().String()) return e.executeWithCallback(input, nil, nil) } switch req.Type() { @@ -205,6 +204,8 @@ func (e *Executer) executeFlow(input *contextargs.Context, callback protocols.Ou allprotos[types.WHOISProtocol.String()] = append(allprotos[types.WHOISProtocol.String()], req) case types.CodeProtocol: allprotos[types.CodeProtocol.String()] = append(allprotos[types.CodeProtocol.String()], req) + default: + gologger.Error().Msgf("invalid request type %s", req.Type().String()) } } flow := &FlowExecutor{ diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 493a9b222d..186aea2223 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" + "github.com/projectdiscovery/nuclei/v2/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps" @@ -32,7 +33,7 @@ type FlowExecutor struct { results *atomic.Bool jsVM *goja.Runtime program *goja.Program - protoFunctions map[string]func(id ...string) // reqFunctions contains functions that allow executing requests/protocols from js + protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js } // Init initializes the flow executor all dependencies @@ -78,7 +79,7 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) // ------ // ---- define callback functions/objects---- - f.protoFunctions = map[string]func(id ...string){} + f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} compileErrors := []error{} for proto, requests := range f.allProtocols { @@ -96,12 +97,18 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) counter++ } // ---define hook that allows protocol/request execution from js----- - f.protoFunctions[proto] = func(ids ...string) { + f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value { defer func() { // to avoid polling update template variables everytime we execute a protocol var m map[string]interface{} = f.options.TemplateCtx.GetAll() _ = f.jsVM.Set("template", m) }() + ids := []string{} + for _, v := range call.Arguments { + ids = append(ids, types.ToString(v.Export())) + } + matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool + // if no id is passed execute all requests in sequence if len(ids) == 0 { // execution logic for http()/dns() etc @@ -114,9 +121,12 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) // export dynamic values from operators (i.e internal:true) // add add it to template context // this is a conflicting behaviour with iterate-all - if result.HasOperatorResult() && len(result.OperatorsResult.DynamicValues) > 0 { - for k, v := range result.OperatorsResult.DynamicValues { - f.options.TemplateCtx.Set(k, v) + if result.HasOperatorResult() { + matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) + if len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.TemplateCtx.Set(k, v) + } } } } @@ -129,10 +139,10 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) id, _ = reqMap.GetKeyWithValue(req) } _ = f.allErrs.Set(id, err) - return + return f.jsVM.ToValue(matcherStatus.Load()) } } - return + return f.jsVM.ToValue(matcherStatus.Load()) } // execution logic for http("0") or http("get-aws-vpcs") @@ -142,7 +152,7 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) gologger.Error().Msgf("invalid request id '%s' provided", id) // compile error compileErrors = append(compileErrors, ErrInvalidRequestID.Msgf(id)) - return + return f.jsVM.ToValue(matcherStatus.Load()) } err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { if result != nil { @@ -150,11 +160,14 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) callback(result) // export dynamic values from operators (i.e internal:true) // add add it to template context - if result.HasOperatorResult() && len(result.OperatorsResult.DynamicValues) > 0 { - for k, v := range result.OperatorsResult.DynamicValues { - f.options.TemplateCtx.Set(k, v) + if result.HasOperatorResult() { + matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) + if len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.TemplateCtx.Set(k, v) + } + _ = f.jsVM.Set("template", f.options.TemplateCtx.GetAll()) } - _ = f.jsVM.Set("template", f.options.TemplateCtx.GetAll()) } } }) @@ -163,6 +176,7 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) _ = f.allErrs.Set(index, err) } } + return f.jsVM.ToValue(matcherStatus.Load()) } } @@ -235,10 +249,13 @@ func (f *FlowExecutor) RegisterBuiltInFunctions() error { // Execute executes the flow func (f *FlowExecutor) Execute() (bool, error) { // pass flow and execute the js vm and handle errors - _, err := f.jsVM.RunProgram(f.program) + value, err := f.jsVM.RunProgram(f.program) if err != nil { return false, errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) } + if value != nil { + return value.ToBoolean(), nil + } return f.results.Load(), nil } diff --git a/v2/pkg/protocols/common/executer/testcases/condition-flow.yaml b/v2/pkg/protocols/common/executer/testcases/condition-flow.yaml new file mode 100644 index 0000000000..6f7098d7f0 --- /dev/null +++ b/v2/pkg/protocols/common/executer/testcases/condition-flow.yaml @@ -0,0 +1,28 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: | + dns() && http("0"); + +dns: + - name: "{{FQDN}}" + type: CNAME + + matchers: + - type: word + words: + - "ghost.io" + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index ab77fe164a..c3107b6496 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -136,25 +136,32 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti switch { case len(template.RequestsDNS) > 0: requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) + fallthrough case len(template.RequestsFile) > 0: requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...) + fallthrough case len(template.RequestsNetwork) > 0: requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) + fallthrough case len(template.RequestsHTTP) > 0: requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + fallthrough case len(template.RequestsHeadless) > 0 && options.Options.Headless: requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) + fallthrough case len(template.RequestsSSL) > 0: requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...) + fallthrough case len(template.RequestsWebsocket) > 0: requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...) + fallthrough case len(template.RequestsWHOIS) > 0: requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) + fallthrough + case len(template.RequestsCode) > 0: + requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...) } } - if len(template.RequestsCode) > 0 { - requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...) - } template.Executer = executer.NewExecuter(requests, &options) return nil } From e329f5c8ba425beee5a5fe9dde744a110d8af582 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 8 Aug 2023 19:44:14 +0530 Subject: [PATCH 15/38] fix proto callbacks + more unit tests --- .../common/executer/flow_executor.go | 5 +- .../common/executer/flow_executor_test.go | 48 +++++++++++++++++++ .../executer/testcases/condition-flow.yaml | 3 +- v2/pkg/templates/compile.go | 35 +++++++------- v2/pkg/templates/templates.go | 2 + 5 files changed, 71 insertions(+), 22 deletions(-) diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 186aea2223..bb0dad893e 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -82,9 +82,10 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} compileErrors := []error{} - for proto, requests := range f.allProtocols { + for p, requests := range f.allProtocols { reqMap := mapsutil.Map[string, protocols.Request]{} counter := 0 + proto := strings.ToLower(p) // donot use loop variables in callback functions directly for index := range requests { request := f.allProtocols[proto][index] if request.GetID() != "" { @@ -253,7 +254,7 @@ func (f *FlowExecutor) Execute() (bool, error) { if err != nil { return false, errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) } - if value != nil { + if value.Export() != nil { return value.ToBoolean(), nil } return f.results.Load(), nil diff --git a/v2/pkg/protocols/common/executer/flow_executor_test.go b/v2/pkg/protocols/common/executer/flow_executor_test.go index 6ecf1d8393..0505be90c9 100644 --- a/v2/pkg/protocols/common/executer/flow_executor_test.go +++ b/v2/pkg/protocols/common/executer/flow_executor_test.go @@ -119,3 +119,51 @@ func TestFlowWithProtoPrefix(t *testing.T) { } } } + +func TestFlowWithConditionNegative(t *testing.T) { + setup() + + // apart from parse->compile->execution this testcase checks + // if bitwise operator (&&) are properly executed and working + Template, err := templates.Parse("testcases/condition-flow.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + // expect no results and verify thant dns request is executed and http is not + gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("scanme.sh")) + require.Nil(t, err, "could not execute template") + require.False(t, gotresults) + + m := Template.Options.TemplateCtx.GetAll() + + require.Equal(t, m["http_status"], nil) // since http() was not execute this variable should not exist + require.NotEqual(t, m["dns_raw"], "") // since dns() was execute this variable should exist +} + +func TestFlowWithConditionPositive(t *testing.T) { + setup() + + // apart from parse->compile->execution this testcase checks + // if bitwise operator (&&) are properly executed and working + Template, err := templates.Parse("testcases/condition-flow.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + // positive match . expect results also verify that both dns() and http() were executed + gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) + + m := Template.Options.TemplateCtx.GetAll() + + require.NotEqual(t, m["http_status"], "") // since http() was not execute this variable should not exist + require.NotEqual(t, m["dns_raw"], "") // since dns() was execute this variable should exist +} diff --git a/v2/pkg/protocols/common/executer/testcases/condition-flow.yaml b/v2/pkg/protocols/common/executer/testcases/condition-flow.yaml index 6f7098d7f0..d1e2cbf9d2 100644 --- a/v2/pkg/protocols/common/executer/testcases/condition-flow.yaml +++ b/v2/pkg/protocols/common/executer/testcases/condition-flow.yaml @@ -5,8 +5,7 @@ info: severity: info -flow: | - dns() && http("0"); +flow: dns() && http() dns: - name: "{{FQDN}}" diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index c3107b6496..964f5f3cf1 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -133,32 +133,31 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti template.MultiProtoRequest.Info = template.Info requests = append(requests, &template.MultiProtoRequest) } else { - switch { - case len(template.RequestsDNS) > 0: + if len(template.RequestsDNS) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) - fallthrough - case len(template.RequestsFile) > 0: + } + if len(template.RequestsFile) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsFile)...) - fallthrough - case len(template.RequestsNetwork) > 0: + } + if len(template.RequestsNetwork) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) - fallthrough - case len(template.RequestsHTTP) > 0: + } + if len(template.RequestsHTTP) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) - fallthrough - case len(template.RequestsHeadless) > 0 && options.Options.Headless: + } + if len(template.RequestsHeadless) > 0 && options.Options.Headless { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) - fallthrough - case len(template.RequestsSSL) > 0: + } + if len(template.RequestsSSL) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsSSL)...) - fallthrough - case len(template.RequestsWebsocket) > 0: + } + if len(template.RequestsWebsocket) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...) - fallthrough - case len(template.RequestsWHOIS) > 0: + } + if len(template.RequestsWHOIS) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) - fallthrough - case len(template.RequestsCode) > 0: + } + if len(template.RequestsCode) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...) } } diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 7a705fb97b..14d14bd158 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -334,6 +334,8 @@ func (template *Template) addProtocolsToQueue(keys ...string) { template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...) case types.WHOISProtocol.String(): template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) + case types.CodeProtocol.String(): + template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsCode)...) } } } From 74e69af06d62ec0c544f182150f6a9fefe4c8dfe Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 9 Aug 2023 21:58:15 +0530 Subject: [PATCH 16/38] adds integration test --- .../flow/conditional-flow-negative.yaml | 27 ++++++ integration_tests/flow/conditional-flow.yaml | 27 ++++++ integration_tests/flow/dns-ns-probe.yaml | 42 +++++++++ .../flow/iterate-values-flow-parallel.yaml | 35 ++++++++ .../flow/iterate-values-flow.yaml | 37 ++++++++ .../protocols/code/py-env-var.yaml | 2 +- integration_tests/protocols/code/py-file.yaml | 2 +- .../protocols/code/py-interactsh.yaml | 2 +- .../protocols/code/py-snippet.yaml | 2 +- v2/cmd/integration-test/flow.go | 85 +++++++++++++++++++ v2/cmd/integration-test/integration-test.go | 1 + v2/cmd/integration-test/multi.go | 4 +- .../common/executer/flow_executor.go | 5 +- v2/pkg/protocols/http/build_request.go | 2 +- v2/pkg/types/interfaces.go | 15 ++++ 15 files changed, 281 insertions(+), 7 deletions(-) create mode 100644 integration_tests/flow/conditional-flow-negative.yaml create mode 100644 integration_tests/flow/conditional-flow.yaml create mode 100644 integration_tests/flow/dns-ns-probe.yaml create mode 100644 integration_tests/flow/iterate-values-flow-parallel.yaml create mode 100644 integration_tests/flow/iterate-values-flow.yaml create mode 100644 v2/cmd/integration-test/flow.go diff --git a/integration_tests/flow/conditional-flow-negative.yaml b/integration_tests/flow/conditional-flow-negative.yaml new file mode 100644 index 0000000000..d1e2cbf9d2 --- /dev/null +++ b/integration_tests/flow/conditional-flow-negative.yaml @@ -0,0 +1,27 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + matchers: + - type: word + words: + - "ghost.io" + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/integration_tests/flow/conditional-flow.yaml b/integration_tests/flow/conditional-flow.yaml new file mode 100644 index 0000000000..d1e2cbf9d2 --- /dev/null +++ b/integration_tests/flow/conditional-flow.yaml @@ -0,0 +1,27 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + matchers: + - type: word + words: + - "ghost.io" + +http: + - method: GET + path: + - "{{BaseURL}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/integration_tests/flow/dns-ns-probe.yaml b/integration_tests/flow/dns-ns-probe.yaml new file mode 100644 index 0000000000..569a9e766c --- /dev/null +++ b/integration_tests/flow/dns-ns-probe.yaml @@ -0,0 +1,42 @@ +id: dns-ns-probe + +info: + name: Nuclei flow dns ns probe + author: pdteam + severity: info + description: Description of the Template + reference: https://example-reference-link + +flow: | + dns("fetch-ns"); + for(let ns of template["nameservers"]) { + set("nameserver",ns); + dns("probe-ns"); + }; + +dns: + - id: "fetch-ns" + name: "{{FQDN}}" + type: NS + matchers: + - type: word + words: + - "IN\tNS" + extractors: + - type: regex + internal: true + name: "nameservers" + group: 1 + regex: + - "IN\tNS\t(.+)" + + - id: "probe-ns" + name: "{{nameserver}}" + type: A + class: inet + retries: 3 + recursion: true + extractors: + - type: dsl + dsl: + - "a" \ No newline at end of file diff --git a/integration_tests/flow/iterate-values-flow-parallel.yaml b/integration_tests/flow/iterate-values-flow-parallel.yaml new file mode 100644 index 0000000000..c6e079acdb --- /dev/null +++ b/integration_tests/flow/iterate-values-flow-parallel.yaml @@ -0,0 +1,35 @@ +id: iterate-values-flow-parallel + +info: + name: Extract Email IDs from Response + author: pdteam + severity: info + + +### this is parallel version of iterate-values-flow.yaml where requests are sent in parallel + +flow: http(0) || http(1) + +http: + - method: GET + path: + - "{{BaseURL}}" + + extractors: + - type: regex + name: emails + internal: true + regex: + - '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' + + - method: GET + path: + - "{{BaseURL}}/user/{{email}}" + + payloads: + email: "{{emails}}" + + matchers: + - type: word + words: + - "Welcome" \ No newline at end of file diff --git a/integration_tests/flow/iterate-values-flow.yaml b/integration_tests/flow/iterate-values-flow.yaml new file mode 100644 index 0000000000..402d4cb912 --- /dev/null +++ b/integration_tests/flow/iterate-values-flow.yaml @@ -0,0 +1,37 @@ +id: extract-emails + +info: + name: Extract Email IDs from Response + author: pdteam + severity: info + + +### this is non-parallel/concurrency=1 version of iterate-values-flow-parallel.yaml + +flow: | + http(0) + for(let email of template["emails"]) { + set("email",email); + http(1); + } + +http: + - method: GET + path: + - "{{BaseURL}}" + + extractors: + - type: regex + name: emails + internal: true + regex: + - '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' + + - method: GET + path: + - "{{BaseURL}}/user/{{base64(email)}}" + + matchers: + - type: word + words: + - "Welcome" \ No newline at end of file diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 9a7b0b3865..95deccc2df 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 490a0046304402205f67e50cb11230efd286c7cef252f09565162375d86298e166d40ca860ca81ff022079652101b5e36405b632f0905bc0350b1d114bcd6b6da47345e454d8fff3ea2d \ No newline at end of file +# digest: 4b0a00483046022100b31b3ea6982d428ca95a2db264695899892baf720338aa0f0ca218f949fc47e0022100bc1df9fcde28b900667a5902b4426f3d790fceab55c5b0668bc4de33d8e7615e \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index daa3491956..991e127e15 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a00483046022100bbc391f379fc1745e56f631d83d74411df6f575ea6213e785069e9fd6f87f41b022100bce8125ba6223aa8fde7703708f02c6b5015cf43043947bb77abca9e1906b524 \ No newline at end of file +# digest: 4a0a00473045022021d642bcce439b172ab0ab0628e423ecf714b7e27a986c3cadab6d5d1ad6e1e4022100cc064029436260e03249b3cec7d2a53ed6d33699a8a7b8620e02681f3e1ff02e \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index 0686b59644..53a31abf95 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 490a00463044022009697d61506d5937791f83dc6407ea023238832593d953ffa53885c105708030022079d88c4bf6d2ebdf241d16c1d16c2b521c8a6332743cdbca4144a9a32b54c6df \ No newline at end of file +# digest: 4a0a00473045022100d70cf3d2ff48c6c4eb0fd45e890e7275bffc92d0cbbbe3f883ff46e470adb011022063997d10a0ff0c2be43471caa3c938376748a9a2436c4b4805ffbcbeba7b9604 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 5611467f08..23847fdb30 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input" -# digest: 4b0a00483046022100e2e8866788f2f225663514736ef03fa7575f97e6a6a6c5963a98c1af49f422c8022100c1a50ae5cdf15cacc4c72bdac72b7101f1cb7fe0ace97fb4f8148de99db6372d \ No newline at end of file +# digest: 490a0046304402201129e869820cadc29821d2cad966b5f6a6cd9b97b5c6d4b95c3f588f85c61dd202203e02fa801b4383d8682e65f0b04ebea421270356c05701a1f0c4bbc513c0f563 \ No newline at end of file diff --git a/v2/cmd/integration-test/flow.go b/v2/cmd/integration-test/flow.go new file mode 100644 index 0000000000..e239f63ec0 --- /dev/null +++ b/v2/cmd/integration-test/flow.go @@ -0,0 +1,85 @@ +package main + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + + "github.com/julienschmidt/httprouter" + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) + +var flowTestcases = []TestCaseInfo{ + {Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}}, + {Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}}, + {Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}}, + {Path: "flow/iterate-values-flow-parallel.yaml", TestCase: &iterateValuesFlow{}}, + {Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}}, +} + +type conditionalFlow struct{} + +func (t *conditionalFlow) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "blog.projectdiscovery.io", debug) + if err != nil { + return err + } + return expectResultsCount(results, 2) +} + +type conditionalFlowNegative struct{} + +func (t *conditionalFlowNegative) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "scanme.sh", debug) + if err != nil { + return err + } + return expectResultsCount(results, 0) +} + +type iterateValuesFlow struct{} + +func (t *iterateValuesFlow) Execute(filePath string) error { + router := httprouter.New() + testemails := []string{ + "secrets@scanme.sh", + "superadmin@scanme.sh", + } + router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprint(testemails))) + }) + router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("Welcome ! This is test matcher text")) + }) + + router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("Welcome ! This is test matcher text")) + }) + + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug) + if err != nil { + return err + } + return expectResultsCount(results, 2) +} + +type dnsNsProbe struct{} + +func (t *dnsNsProbe) Execute(filePath string) error { + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, "oast.fun", debug) + if err != nil { + return err + } + return expectResultsCount(results, 3) +} + +func getBase64(input string) string { + return base64.StdEncoding.EncodeToString([]byte(input)) +} diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go index 66d562ccff..f18560c773 100644 --- a/v2/cmd/integration-test/integration-test.go +++ b/v2/cmd/integration-test/integration-test.go @@ -49,6 +49,7 @@ var ( "multi": multiProtoTestcases, "generic": genericTestcases, "dsl": dslTestcases, + "flow": flowTestcases, } // For debug purposes diff --git a/v2/cmd/integration-test/multi.go b/v2/cmd/integration-test/multi.go index a7998ab358..a9ff58fffd 100644 --- a/v2/cmd/integration-test/multi.go +++ b/v2/cmd/integration-test/multi.go @@ -1,6 +1,8 @@ package main -import "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/testutils" +) var multiProtoTestcases = []TestCaseInfo{ {Path: "protocols/multi/dynamic-values.yaml", TestCase: &multiProtoDynamicExtractor{}}, diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index bb0dad893e..a774060513 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -225,7 +225,10 @@ func (f *FlowExecutor) RegisterBuiltInFunctions() error { return err } - var m map[string]interface{} = f.options.TemplateCtx.GetAll() + var m = f.options.TemplateCtx.GetAll() + if m == nil { + m = map[string]interface{}{} + } if err := f.jsVM.Set("template", m); err != nil { // all template variables are available in js template object diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 18595b0cbb..06bccc252f 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -86,7 +86,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, } } else { for payloadName, payloadValue := range payloads { - payloads[payloadName] = types.ToString(payloadValue) + payloads[payloadName] = types.ToStringNSlice(payloadValue) } } diff --git a/v2/pkg/types/interfaces.go b/v2/pkg/types/interfaces.go index 25dd685677..7b5bcd2ae8 100644 --- a/v2/pkg/types/interfaces.go +++ b/v2/pkg/types/interfaces.go @@ -77,6 +77,21 @@ func ToString(data interface{}) string { } } +// ToStringNSlice converts an interface to string in a quick way or to a slice with strings +// if the input is a slice of interfaces. +func ToStringNSlice(data interface{}) interface{} { + switch s := data.(type) { + case []interface{}: + var a []string + for _, v := range s { + a = append(a, ToString(v)) + } + return a + default: + return ToString(data) + } +} + func ToHexOrString(data interface{}) string { switch s := data.(type) { case string: From 64098b6567a2d6b7fc3e376d61af73836b3277bb Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 9 Aug 2023 22:07:19 +0530 Subject: [PATCH 17/38] conditional flow: check if req has any matchers --- .../flow/iterate-values-flow-parallel.yaml | 35 ------------------- .../flow/iterate-values-flow.yaml | 2 -- .../protocols/code/py-env-var.yaml | 2 +- integration_tests/protocols/code/py-file.yaml | 2 +- .../protocols/code/py-interactsh.yaml | 2 +- .../protocols/code/py-snippet.yaml | 2 +- v2/cmd/integration-test/flow.go | 1 - .../common/executer/flow_executor.go | 16 +++++++++ 8 files changed, 20 insertions(+), 42 deletions(-) delete mode 100644 integration_tests/flow/iterate-values-flow-parallel.yaml diff --git a/integration_tests/flow/iterate-values-flow-parallel.yaml b/integration_tests/flow/iterate-values-flow-parallel.yaml deleted file mode 100644 index c6e079acdb..0000000000 --- a/integration_tests/flow/iterate-values-flow-parallel.yaml +++ /dev/null @@ -1,35 +0,0 @@ -id: iterate-values-flow-parallel - -info: - name: Extract Email IDs from Response - author: pdteam - severity: info - - -### this is parallel version of iterate-values-flow.yaml where requests are sent in parallel - -flow: http(0) || http(1) - -http: - - method: GET - path: - - "{{BaseURL}}" - - extractors: - - type: regex - name: emails - internal: true - regex: - - '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' - - - method: GET - path: - - "{{BaseURL}}/user/{{email}}" - - payloads: - email: "{{emails}}" - - matchers: - - type: word - words: - - "Welcome" \ No newline at end of file diff --git a/integration_tests/flow/iterate-values-flow.yaml b/integration_tests/flow/iterate-values-flow.yaml index 402d4cb912..3abea9e4cb 100644 --- a/integration_tests/flow/iterate-values-flow.yaml +++ b/integration_tests/flow/iterate-values-flow.yaml @@ -6,8 +6,6 @@ info: severity: info -### this is non-parallel/concurrency=1 version of iterate-values-flow-parallel.yaml - flow: | http(0) for(let email of template["emails"]) { diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 95deccc2df..7ee2654bcc 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4b0a00483046022100b31b3ea6982d428ca95a2db264695899892baf720338aa0f0ca218f949fc47e0022100bc1df9fcde28b900667a5902b4426f3d790fceab55c5b0668bc4de33d8e7615e \ No newline at end of file +# digest: 4b0a00483046022100bb80bf0ba533906fe86a0436b427d46e1ebe9627054d7e334b017c7532909b8a0221009346e92aea683e1fea54195081abf084ca5c2ade85566d610aea4e2368d07526 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 991e127e15..99294d74f8 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a00473045022021d642bcce439b172ab0ab0628e423ecf714b7e27a986c3cadab6d5d1ad6e1e4022100cc064029436260e03249b3cec7d2a53ed6d33699a8a7b8620e02681f3e1ff02e \ No newline at end of file +# digest: 4a0a0047304502207ef2ec2a351f2d0bc77bb4792abd297cdb6e8735a45669a3d0848a3d833a47b7022100d3f9e0d5017c0c465fedc999d14ae61ba64c804a04d281bac540faa321f4ae74 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index 53a31abf95..2b99fabdbb 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4a0a00473045022100d70cf3d2ff48c6c4eb0fd45e890e7275bffc92d0cbbbe3f883ff46e470adb011022063997d10a0ff0c2be43471caa3c938376748a9a2436c4b4805ffbcbeba7b9604 \ No newline at end of file +# digest: 4a0a0047304502207de0206dcd3be754fa65f2c2f93106faa5c4b5d9e04b521b12c5baabe3e5246c022100d2eccbe8e135fb55b93555028b8d92928a50afb1ddb20137d891fe53ca7048e1 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 23847fdb30..59ac2fe0b5 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input" -# digest: 490a0046304402201129e869820cadc29821d2cad966b5f6a6cd9b97b5c6d4b95c3f588f85c61dd202203e02fa801b4383d8682e65f0b04ebea421270356c05701a1f0c4bbc513c0f563 \ No newline at end of file +# digest: 490a0046304402202557bb272aca2b21cfafa5cd82b519dd23d4490b2f34ed602529508fb1aeeead02207c613b01ce297d8b60295a3ba2318787564518ab76fef2d013d96e26f08829c6 \ No newline at end of file diff --git a/v2/cmd/integration-test/flow.go b/v2/cmd/integration-test/flow.go index e239f63ec0..73109598d1 100644 --- a/v2/cmd/integration-test/flow.go +++ b/v2/cmd/integration-test/flow.go @@ -14,7 +14,6 @@ var flowTestcases = []TestCaseInfo{ {Path: "flow/conditional-flow.yaml", TestCase: &conditionalFlow{}}, {Path: "flow/conditional-flow-negative.yaml", TestCase: &conditionalFlowNegative{}}, {Path: "flow/iterate-values-flow.yaml", TestCase: &iterateValuesFlow{}}, - {Path: "flow/iterate-values-flow-parallel.yaml", TestCase: &iterateValuesFlow{}}, {Path: "flow/dns-ns-probe.yaml", TestCase: &dnsNsProbe{}}, } diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index a774060513..f6b6d600bb 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -9,6 +9,7 @@ import ( "github.com/dop251/goja" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" @@ -124,6 +125,11 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) // this is a conflicting behaviour with iterate-all if result.HasOperatorResult() { matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) + if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) { + // if matcher status is false . check if template/request contains any matcher at all + // if it does then we need to set matcher status to true + matcherStatus.CompareAndSwap(false, true) + } if len(result.OperatorsResult.DynamicValues) > 0 { for k, v := range result.OperatorsResult.DynamicValues { f.options.TemplateCtx.Set(k, v) @@ -284,3 +290,13 @@ func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { } return values, nil } + +// Checks if template has matchers +func hasMatchers(all []*operators.Operators) bool { + for _, operator := range all { + if len(operator.Matchers) > 0 { + return true + } + } + return false +} From aa5da2bc31e52f8c733c2e2035ee6699caac2cfa Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 9 Aug 2023 22:10:09 +0530 Subject: [PATCH 18/38] fix lint error --- v2/cmd/integration-test/flow.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/cmd/integration-test/flow.go b/v2/cmd/integration-test/flow.go index 73109598d1..67d4b749b4 100644 --- a/v2/cmd/integration-test/flow.go +++ b/v2/cmd/integration-test/flow.go @@ -47,16 +47,16 @@ func (t *iterateValuesFlow) Execute(filePath string) error { } router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { w.WriteHeader(http.StatusOK) - w.Write([]byte(fmt.Sprint(testemails))) + _, _ = w.Write([]byte(fmt.Sprint(testemails))) }) router.GET("/user/"+getBase64(testemails[0]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { w.WriteHeader(http.StatusOK) - w.Write([]byte("Welcome ! This is test matcher text")) + _, _ = w.Write([]byte("Welcome ! This is test matcher text")) }) router.GET("/user/"+getBase64(testemails[1]), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { w.WriteHeader(http.StatusOK) - w.Write([]byte("Welcome ! This is test matcher text")) + _, _ = w.Write([]byte("Welcome ! This is test matcher text")) }) ts := httptest.NewServer(router) From ab36aeed39fc7dd507aa1e09d2dd64a24f24a451 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Thu, 10 Aug 2023 19:28:05 +0530 Subject: [PATCH 19/38] deprecate iterate-all+ missing multi-proto implementation --- v2/pkg/protocols/code/code.go | 11 ++++++++++- v2/pkg/protocols/http/http.go | 1 + v2/pkg/protocols/multi/README.md | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/v2/pkg/protocols/code/code.go b/v2/pkg/protocols/code/code.go index e7f710074b..14de35becb 100644 --- a/v2/pkg/protocols/code/code.go +++ b/v2/pkg/protocols/code/code.go @@ -117,10 +117,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa // inject all template context values as gozero env variables variables := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil) + // add template context values + variables = generators.MergeMaps(variables, request.options.TemplateCtx.GetAll()) // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) variablesMap := request.options.Variables.Evaluate(variables) - variables = generators.MergeMaps(variablesMap, variables, optionVars) + variables = generators.MergeMaps(variablesMap, variables, optionVars, request.options.Constants) for name, value := range variables { v := fmt.Sprint(value) v, interactshURLs = request.options.Interactsh.Replace(v, interactshURLs) @@ -153,6 +155,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa data["template-id"] = request.options.TemplateID data["template-info"] = request.options.TemplateInfo + // expose response variables in proto_var format + // this is no-op if the template is not a multi protocol template + request.options.AddTemplateVars(request.Type(), request.ID, data) + + // add variables from template context before matching/extraction + data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) + if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(interactshURLs, data) } diff --git a/v2/pkg/protocols/http/http.go b/v2/pkg/protocols/http/http.go index 7cef0a2e18..afd5545a17 100644 --- a/v2/pkg/protocols/http/http.go +++ b/v2/pkg/protocols/http/http.go @@ -190,6 +190,7 @@ type Request struct { SkipVariablesCheck bool `yaml:"skip-variables-check,omitempty" json:"skip-variables-check,omitempty" jsonschema:"title=skip variable checks,description=Skips the check for unresolved variables in request"` // description: | // IterateAll iterates all the values extracted from internal extractors + // Deprecated: Use flow instead . iterate-all will be removed in future releases IterateAll bool `yaml:"iterate-all,omitempty" json:"iterate-all,omitempty" jsonschema:"title=iterate all the values,description=Iterates all the values extracted from internal extractors"` // description: | // DigestAuthUsername specifies the username for digest authentication diff --git a/v2/pkg/protocols/multi/README.md b/v2/pkg/protocols/multi/README.md index f7c665cdb8..f905554001 100644 --- a/v2/pkg/protocols/multi/README.md +++ b/v2/pkg/protocols/multi/README.md @@ -36,7 +36,7 @@ to make response fields of that protocol available to global context outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData) // expose response variables in proto_var format // this is no-op if the template is not a multi protocol template - request.options.AddTemplateVars(request.Type(), outputEvent) + request.options.AddTemplateVars(request.Type(),request.ID, outputEvent) ``` - Append all available template context values to outputEvent From 02f4237dbeac96a4820ec96230c007d6a9105ec2 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 14 Aug 2023 00:50:17 +0530 Subject: [PATCH 20/38] fix ip input in raw request --- .../common/executer/flow_executor.go | 23 +++++++++++++++++-- v2/pkg/protocols/http/raw/raw.go | 5 +++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index f6b6d600bb..e5fabba15e 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -138,6 +138,7 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) } } }) + // fmt.Printf("done executing %v with index %v and err %v", proto, index, err) if err != nil { // save all errors in a map with id as key // its less likely that there will be race condition but just in case @@ -145,7 +146,10 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) if id == "" { id, _ = reqMap.GetKeyWithValue(req) } - _ = f.allErrs.Set(id, err) + err = f.allErrs.Set(proto+":"+id, err) + if err != nil { + gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + } return f.jsVM.ToValue(matcherStatus.Load()) } } @@ -180,7 +184,10 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) }) if err != nil { index := id - _ = f.allErrs.Set(index, err) + err = f.allErrs.Set(proto+":"+index, err) + if err != nil { + gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + } } } return f.jsVM.ToValue(matcherStatus.Load()) @@ -263,12 +270,24 @@ func (f *FlowExecutor) Execute() (bool, error) { if err != nil { return false, errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) } + runtimeErr := f.GetRuntimeErrors() + if runtimeErr != nil { + return false, errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") + } if value.Export() != nil { return value.ToBoolean(), nil } return f.results.Load(), nil } +func (f *FlowExecutor) GetRuntimeErrors() error { + errs := []error{} + for proto, err := range f.allErrs.GetAll() { + errs = append(errs, errorutil.NewWithErr(err).Msgf("failed to execute %v protocol", proto)) + } + return multierr.Combine(errs...) +} + // ReadDataFromFile reads data from file respecting sandbox options func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { values := []string{} diff --git a/v2/pkg/protocols/http/raw/raw.go b/v2/pkg/protocols/http/raw/raw.go index 58aae6cb82..c65ffe38b6 100644 --- a/v2/pkg/protocols/http/raw/raw.go +++ b/v2/pkg/protocols/http/raw/raw.go @@ -102,7 +102,10 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b if _, ok := rawrequest.Headers["Host"]; !ok { rawrequest.Headers["Host"] = inputURL.Host } - rawrequest.FullURL = fmt.Sprintf("%s://%s%s", inputURL.Scheme, strings.TrimSpace(inputURL.Host), rawrequest.Path) + cloned := inputURL.Clone() + cloned.Path = "" + _ = cloned.MergePath(rawrequest.Path, true) + rawrequest.FullURL = cloned.String() } return rawrequest, nil From 9c79cb4f27375be2477e47616bbb775cb8154b9f Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 14 Aug 2023 02:33:31 +0530 Subject: [PATCH 21/38] JS: feat dedupe object+ more builtin funcs --- .../common/executer/builtin/dedupe.go | 67 ++++++++++++++++++ .../common/executer/flow_executor.go | 70 +++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 v2/pkg/protocols/common/executer/builtin/dedupe.go diff --git a/v2/pkg/protocols/common/executer/builtin/dedupe.go b/v2/pkg/protocols/common/executer/builtin/dedupe.go new file mode 100644 index 0000000000..eae088db8a --- /dev/null +++ b/v2/pkg/protocols/common/executer/builtin/dedupe.go @@ -0,0 +1,67 @@ +package builtin + +import ( + "crypto/md5" + "reflect" + + "github.com/dop251/goja" + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// Dedupe is a javascript builtin for deduping values +type Dedupe struct { + m map[string]goja.Value + VM *goja.Runtime +} + +// Add adds a value to the dedupe +func (d *Dedupe) Add(call goja.FunctionCall) goja.Value { + allVars := []any{} + for _, v := range call.Arguments { + if v.Export() == nil { + continue + } + if v.ExportType().Kind() == reflect.Slice { + // convert []datatype to []interface{} + // since it cannot be type asserted to []interface{} directly + rfValue := reflect.ValueOf(v.Export()) + for i := 0; i < rfValue.Len(); i++ { + allVars = append(allVars, rfValue.Index(i).Interface()) + } + } else { + allVars = append(allVars, v.Export()) + } + } + for _, v := range allVars { + hash := hashValue(v) + if _, ok := d.m[hash]; ok { + continue + } + d.m[hash] = d.VM.ToValue(v) + } + return d.VM.ToValue(true) +} + +// Values returns all values from the dedupe +func (d *Dedupe) Values(call goja.FunctionCall) goja.Value { + tmp := []goja.Value{} + for _, v := range d.m { + tmp = append(tmp, v) + } + return d.VM.ToValue(tmp) +} + +// NewDedupe creates a new dedupe builtin object +func NewDedupe(vm *goja.Runtime) *Dedupe { + return &Dedupe{ + m: make(map[string]goja.Value), + VM: vm, + } +} + +// hashValue returns a hash of the value +func hashValue(value interface{}) string { + res := types.ToString(value) + md5sum := md5.Sum([]byte(res)) + return string(md5sum[:]) +} diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index e5fabba15e..774f7ce1c0 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -2,6 +2,7 @@ package executer import ( "io" + "reflect" "strconv" "strings" "sync/atomic" @@ -13,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer/builtin" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v2/pkg/types" @@ -238,6 +240,74 @@ func (f *FlowExecutor) RegisterBuiltInFunctions() error { return err } + // iterate provides global iterator function by handling null values or strings + if err := f.jsVM.Set("iterate", func(call goja.FunctionCall) goja.Value { + allVars := []any{} + for _, v := range call.Arguments { + if v.Export() == nil { + continue + } + if v.ExportType().Kind() == reflect.Slice { + // convert []datatype to []interface{} + // since it cannot be type asserted to []interface{} directly + rfValue := reflect.ValueOf(v.Export()) + for i := 0; i < rfValue.Len(); i++ { + allVars = append(allVars, rfValue.Index(i).Interface()) + } + } else { + allVars = append(allVars, v.Export()) + } + } + return f.jsVM.ToValue(allVars) + }); err != nil { + return err + } + + // unfortunately js doesn't have trimLeft/trimRight + if err := f.jsVM.Set("trimLeft", func(call goja.FunctionCall) goja.Value { + value := call.Argument(0).String() + char := call.Argument(1).String() + if char == "" { + char = " " + } + return f.jsVM.ToValue(strings.TrimLeft(value, char)) + }); err != nil { + return err + } + + if err := f.jsVM.Set("trimRight", func(call goja.FunctionCall) goja.Value { + value := call.Argument(0).String() + char := call.Argument(1).String() + if char == "" { + char = " " + } + return f.jsVM.ToValue(strings.TrimRight(value, char)) + }); err != nil { + return err + } + + if err := f.jsVM.Set("trim", func(call goja.FunctionCall) goja.Value { + value := call.Argument(0).String() + char := call.Argument(1).String() + if char == "" { + char = " " + } + return f.jsVM.ToValue(strings.Trim(value, char)) + }); err != nil { + return err + } + + // add a builtin dedupe object + if err := f.jsVM.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { + d := builtin.NewDedupe(f.jsVM) + obj := call.This + obj.Set("Add", d.Add) + obj.Set("Values", d.Values) + return nil + }); err != nil { + return err + } + var m = f.options.TemplateCtx.GetAll() if m == nil { m = map[string]interface{}{} From 787fe5e01c7e05e5b1c93c0792f35a9fb5814c3f Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 14 Aug 2023 02:57:09 +0530 Subject: [PATCH 22/38] feat: hide protocol result using hide --- .../common/executer/flow_executor.go | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 774f7ce1c0..60ed8d5949 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -28,6 +28,17 @@ var ( ErrInvalidRequestID = errorutil.NewWithFmt("invalid request id '%s' provided") ) +type ProtoOptions struct { + Hide bool +} + +func GetProtoOptions(m map[string]interface{}) *ProtoOptions { + options := &ProtoOptions{ + Hide: GetBool(m["hide"]), + } + return options +} + type FlowExecutor struct { input *contextargs.Context allProtocols map[string][]protocols.Request @@ -108,8 +119,14 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) _ = f.jsVM.Set("template", m) }() ids := []string{} + opts := &ProtoOptions{} for _, v := range call.Arguments { - ids = append(ids, types.ToString(v.Export())) + switch value := v.Export().(type) { + case map[string]interface{}: + opts = GetProtoOptions(value) + default: + ids = append(ids, types.ToString(value)) + } } matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool @@ -121,7 +138,9 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { if result != nil { f.results.CompareAndSwap(false, true) - callback(result) + if !opts.Hide { + callback(result) + } // export dynamic values from operators (i.e internal:true) // add add it to template context // this is a conflicting behaviour with iterate-all @@ -170,7 +189,9 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { if result != nil { f.results.CompareAndSwap(false, true) - callback(result) + if !opts.Hide { + callback(result) + } // export dynamic values from operators (i.e internal:true) // add add it to template context if result.HasOperatorResult() { @@ -389,3 +410,20 @@ func hasMatchers(all []*operators.Operators) bool { } return false } + +// GetBool returns bool value from interface +func GetBool(value interface{}) bool { + if value == nil { + return false + } + switch v := value.(type) { + case bool: + return v + default: + tmpValue := types.ToString(value) + if strings.EqualFold(tmpValue, "true") { + return true + } + } + return false +} From f4abe7a725c28cd658c5abe8451f9970413826ae Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 14 Aug 2023 03:27:46 +0530 Subject: [PATCH 23/38] feat: async execution --- .../common/executer/flow_executor.go | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 60ed8d5949..0819752b4e 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -5,6 +5,7 @@ import ( "reflect" "strconv" "strings" + "sync" "sync/atomic" "github.com/dop251/goja" @@ -29,12 +30,14 @@ var ( ) type ProtoOptions struct { - Hide bool + Hide bool + Async bool } func GetProtoOptions(m map[string]interface{}) *ProtoOptions { options := &ProtoOptions{ - Hide: GetBool(m["hide"]), + Hide: GetBool(m["hide"]), + Async: GetBool(m["async"]), } return options } @@ -48,6 +51,7 @@ type FlowExecutor struct { jsVM *goja.Runtime program *goja.Program protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js + wg sync.WaitGroup } // Init initializes the flow executor all dependencies @@ -113,11 +117,6 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) } // ---define hook that allows protocol/request execution from js----- f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value { - defer func() { - // to avoid polling update template variables everytime we execute a protocol - var m map[string]interface{} = f.options.TemplateCtx.GetAll() - _ = f.jsVM.Set("template", m) - }() ids := []string{} opts := &ProtoOptions{} for _, v := range call.Arguments { @@ -128,6 +127,15 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) ids = append(ids, types.ToString(value)) } } + if opts.Async { + f.wg.Add(1) + defer f.wg.Done() + } + defer func() { + // to avoid polling update template variables everytime we execute a protocol + var m map[string]interface{} = f.options.TemplateCtx.GetAll() + _ = f.jsVM.Set("template", m) + }() matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool // if no id is passed execute all requests in sequence @@ -361,6 +369,7 @@ func (f *FlowExecutor) Execute() (bool, error) { if err != nil { return false, errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) } + f.wg.Wait() runtimeErr := f.GetRuntimeErrors() if runtimeErr != nil { return false, errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") From 2a329338f73b90269a72fcd62ae2581cdf060c6c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 14 Aug 2023 04:07:37 +0530 Subject: [PATCH 24/38] complete async execution support --- v2/pkg/protocols/common/executer/executer.go | 1 + .../common/executer/flow_executor.go | 218 +++++++++--------- 2 files changed, 115 insertions(+), 104 deletions(-) diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index 6b61833b49..94905003bd 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -208,6 +208,7 @@ func (e *Executer) executeFlow(input *contextargs.Context, callback protocols.Ou gologger.Error().Msgf("invalid request type %s", req.Type().String()) } } + flow := &FlowExecutor{ allProtocols: allprotos, input: input, diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 0819752b4e..4cdd276aa5 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -30,16 +30,17 @@ var ( ) type ProtoOptions struct { - Hide bool - Async bool + Hide bool + Async bool + protoName string + reqIDS []string + callback func(result *output.InternalWrappedEvent) } -func GetProtoOptions(m map[string]interface{}) *ProtoOptions { - options := &ProtoOptions{ - Hide: GetBool(m["hide"]), - Async: GetBool(m["async"]), - } - return options +// LoadOptions loads the protocol options from a map +func (P *ProtoOptions) LoadOptions(m map[string]interface{}) { + P.Hide = GetBool(m["hide"]) + P.Async = GetBool(m["async"]) } type FlowExecutor struct { @@ -94,13 +95,12 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) } } f.options.TemplateCtx.Merge(allVars) - // ------ // ---- define callback functions/objects---- f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} - compileErrors := []error{} - + // iterate over all protocols and generate callback functions for each protocol for p, requests := range f.allProtocols { + // for each protocol build a requestMap with reqID and protocol request reqMap := mapsutil.Map[string, protocols.Request]{} counter := 0 proto := strings.ToLower(p) // donot use loop variables in callback functions directly @@ -116,121 +116,130 @@ func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent) counter++ } // ---define hook that allows protocol/request execution from js----- + // --- this is the actual callback that is executed when function is invoked in js---- f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value { - ids := []string{} - opts := &ProtoOptions{} + opts := &ProtoOptions{ + callback: callback, + protoName: proto, + } for _, v := range call.Arguments { switch value := v.Export().(type) { case map[string]interface{}: - opts = GetProtoOptions(value) + opts.LoadOptions(value) default: - ids = append(ids, types.ToString(value)) + opts.reqIDS = append(opts.reqIDS, types.ToString(value)) } } if opts.Async { f.wg.Add(1) - defer f.wg.Done() + go func() { + defer f.wg.Done() + f.requestExecutor(reqMap, opts) + }() + return f.jsVM.ToValue(true) } - defer func() { - // to avoid polling update template variables everytime we execute a protocol - var m map[string]interface{} = f.options.TemplateCtx.GetAll() - _ = f.jsVM.Set("template", m) - }() - matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool - // if no id is passed execute all requests in sequence - if len(ids) == 0 { - // execution logic for http()/dns() etc - for index := range f.allProtocols[proto] { - req := f.allProtocols[proto][index] - err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { - if result != nil { - f.results.CompareAndSwap(false, true) - if !opts.Hide { - callback(result) - } - // export dynamic values from operators (i.e internal:true) - // add add it to template context - // this is a conflicting behaviour with iterate-all - if result.HasOperatorResult() { - matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) - if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) { - // if matcher status is false . check if template/request contains any matcher at all - // if it does then we need to set matcher status to true - matcherStatus.CompareAndSwap(false, true) - } - if len(result.OperatorsResult.DynamicValues) > 0 { - for k, v := range result.OperatorsResult.DynamicValues { - f.options.TemplateCtx.Set(k, v) - } - } - } - } - }) - // fmt.Printf("done executing %v with index %v and err %v", proto, index, err) - if err != nil { - // save all errors in a map with id as key - // its less likely that there will be race condition but just in case - id := req.GetID() - if id == "" { - id, _ = reqMap.GetKeyWithValue(req) + return f.jsVM.ToValue(f.requestExecutor(reqMap, opts)) + } + } + + // register all built in functions + return f.RegisterBuiltInFunctions() +} + +// requestExecutor executes a protocol/request and returns true if any matcher was found +func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool { + defer func() { + // to avoid polling update template variables everytime we execute a protocol + var m map[string]interface{} = f.options.TemplateCtx.GetAll() + _ = f.jsVM.Set("template", m) + }() + matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool + // if no id is passed execute all requests in sequence + if len(opts.reqIDS) == 0 { + // execution logic for http()/dns() etc + for index := range f.allProtocols[opts.protoName] { + req := f.allProtocols[opts.protoName][index] + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { + if result != nil { + f.results.CompareAndSwap(false, true) + if !opts.Hide { + opts.callback(result) + } + // export dynamic values from operators (i.e internal:true) + // add add it to template context + // this is a conflicting behaviour with iterate-all + if result.HasOperatorResult() { + matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) + if !result.OperatorsResult.Matched && !hasMatchers(req.GetCompiledOperators()) { + // if matcher status is false . check if template/request contains any matcher at all + // if it does then we need to set matcher status to true + matcherStatus.CompareAndSwap(false, true) } - err = f.allErrs.Set(proto+":"+id, err) - if err != nil { - gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + if len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.TemplateCtx.Set(k, v) + } } - return f.jsVM.ToValue(matcherStatus.Load()) } } - return f.jsVM.ToValue(matcherStatus.Load()) + }) + if err != nil { + // save all errors in a map with id as key + // its less likely that there will be race condition but just in case + id := req.GetID() + if id == "" { + id, _ = reqMap.GetKeyWithValue(req) + } + err = f.allErrs.Set(opts.protoName+":"+id, err) + if err != nil { + gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + } + return matcherStatus.Load() } + } + return matcherStatus.Load() + } - // execution logic for http("0") or http("get-aws-vpcs") - for _, id := range ids { - req, ok := reqMap[id] - if !ok { - gologger.Error().Msgf("invalid request id '%s' provided", id) - // compile error - compileErrors = append(compileErrors, ErrInvalidRequestID.Msgf(id)) - return f.jsVM.ToValue(matcherStatus.Load()) + // execution logic for http("0") or http("get-aws-vpcs") + for _, id := range opts.reqIDS { + req, ok := reqMap[id] + if !ok { + gologger.Error().Msgf("invalid request id '%s' provided", id) + // compile error + if err := f.allErrs.Set(opts.protoName+":"+id, ErrInvalidRequestID.Msgf(id)); err != nil { + gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + } + return matcherStatus.Load() + } + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { + if result != nil { + f.results.CompareAndSwap(false, true) + if !opts.Hide { + opts.callback(result) } - err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { - if result != nil { - f.results.CompareAndSwap(false, true) - if !opts.Hide { - callback(result) - } - // export dynamic values from operators (i.e internal:true) - // add add it to template context - if result.HasOperatorResult() { - matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) - if len(result.OperatorsResult.DynamicValues) > 0 { - for k, v := range result.OperatorsResult.DynamicValues { - f.options.TemplateCtx.Set(k, v) - } - _ = f.jsVM.Set("template", f.options.TemplateCtx.GetAll()) - } + // export dynamic values from operators (i.e internal:true) + // add add it to template context + if result.HasOperatorResult() { + matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) + if len(result.OperatorsResult.DynamicValues) > 0 { + for k, v := range result.OperatorsResult.DynamicValues { + f.options.TemplateCtx.Set(k, v) } - } - }) - if err != nil { - index := id - err = f.allErrs.Set(proto+":"+index, err) - if err != nil { - gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + _ = f.jsVM.Set("template", f.options.TemplateCtx.GetAll()) } } } - return f.jsVM.ToValue(matcherStatus.Load()) + }) + if err != nil { + index := id + err = f.allErrs.Set(opts.protoName+":"+index, err) + if err != nil { + gologger.Error().Msgf("failed to store flow runtime errors got %v", err) + } } } - - if len(compileErrors) > 0 { - return multierr.Combine(compileErrors...) - } - - // register all built in functions - return f.RegisterBuiltInFunctions() + return matcherStatus.Load() } // RegisterBuiltInFunctions registers all built in functions for the flow @@ -330,8 +339,9 @@ func (f *FlowExecutor) RegisterBuiltInFunctions() error { if err := f.jsVM.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { d := builtin.NewDedupe(f.jsVM) obj := call.This - obj.Set("Add", d.Add) - obj.Set("Values", d.Values) + // register these methods + _ = obj.Set("Add", d.Add) + _ = obj.Set("Values", d.Values) return nil }); err != nil { return err From f4150de275bfcdb19a8292a5204775a852a81e6d Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 21 Aug 2023 19:35:50 +0530 Subject: [PATCH 25/38] fix condition-flow without any matchers --- .../common/executer/flow_executor.go | 14 ++++++++ .../common/executer/flow_executor_test.go | 34 +++++++++++++++++++ .../testcases/condition-flow-extractors.yaml | 29 ++++++++++++++++ .../condition-flow-no-operators.yaml | 23 +++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 v2/pkg/protocols/common/executer/testcases/condition-flow-extractors.yaml create mode 100644 v2/pkg/protocols/common/executer/testcases/condition-flow-no-operators.yaml diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/protocols/common/executer/flow_executor.go index 4cdd276aa5..9cd4b8ab70 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/protocols/common/executer/flow_executor.go @@ -181,6 +181,10 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req f.options.TemplateCtx.Set(k, v) } } + } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { + // if matcher status is false . check if template/request contains any matcher at all + // if it does then we need to set matcher status to true + matcherStatus.CompareAndSwap(false, true) } } }) @@ -430,6 +434,16 @@ func hasMatchers(all []*operators.Operators) bool { return false } +// hasOperators checks if template has operators (i.e matchers/extractors) +func hasOperators(all []*operators.Operators) bool { + for _, operator := range all { + if operator != nil { + return true + } + } + return false +} + // GetBool returns bool value from interface func GetBool(value interface{}) bool { if value == nil { diff --git a/v2/pkg/protocols/common/executer/flow_executor_test.go b/v2/pkg/protocols/common/executer/flow_executor_test.go index 0505be90c9..e50b6cd2aa 100644 --- a/v2/pkg/protocols/common/executer/flow_executor_test.go +++ b/v2/pkg/protocols/common/executer/flow_executor_test.go @@ -167,3 +167,37 @@ func TestFlowWithConditionPositive(t *testing.T) { require.NotEqual(t, m["http_status"], "") // since http() was not execute this variable should not exist require.NotEqual(t, m["dns_raw"], "") // since dns() was execute this variable should exist } + +func TestFlowWithNoMatchers(t *testing.T) { + // when using conditional flow with no matchers at all + // we implicitly assume that request was successful and internally changed the result to true (for scope of condition only) + + // testcase-1 : no matchers but contains extractor + Template, err := templates.Parse("testcases/condition-flow-extractors.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + // positive match . expect results also verify that both dns() and http() were executed + gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) + + // testcase-2 : no matchers and no extractors + Template, err = templates.Parse("testcases/condition-flow-no-operators.yaml", nil, executerOpts) + require.Nil(t, err, "could not parse template") + + require.True(t, Template.Flow != "", "not a flow template") // this is classifer if template is flow or not + + err = Template.Executer.Compile() + require.Nil(t, err, "could not compile template") + + // positive match . expect results also verify that both dns() and http() were executed + gotresults, err = Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + require.Nil(t, err, "could not execute template") + require.True(t, gotresults) + +} diff --git a/v2/pkg/protocols/common/executer/testcases/condition-flow-extractors.yaml b/v2/pkg/protocols/common/executer/testcases/condition-flow-extractors.yaml new file mode 100644 index 0000000000..8dcb7c4f06 --- /dev/null +++ b/v2/pkg/protocols/common/executer/testcases/condition-flow-extractors.yaml @@ -0,0 +1,29 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + +dns: + - name: "{{FQDN}}" + type: CNAME + + extractors: + - type: dsl + name: cname + internal: true + dsl: + - cname + +http: + - method: GET + path: + - "{{BaseURL}}?ref={{cname}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file diff --git a/v2/pkg/protocols/common/executer/testcases/condition-flow-no-operators.yaml b/v2/pkg/protocols/common/executer/testcases/condition-flow-no-operators.yaml new file mode 100644 index 0000000000..8cb687b248 --- /dev/null +++ b/v2/pkg/protocols/common/executer/testcases/condition-flow-no-operators.yaml @@ -0,0 +1,23 @@ +id: ghost-blog-detection +info: + name: Ghost blog detection + author: pdteam + severity: info + + +flow: dns() && http() + + +dns: + - name: "{{FQDN}}" + type: CNAME + +http: + - method: GET + path: + - "{{BaseURL}}?ref={{dns_cname}}" + + matchers: + - type: word + words: + - "ghost.io" \ No newline at end of file From 3f4e885c2e50d51b2eae9d5480eb603ca0866369 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 21 Aug 2023 21:29:29 +0530 Subject: [PATCH 26/38] refactor: template executer package (tmplexec) --- v2/pkg/protocols/common/executer/executer.go | 228 ----------------- .../headless/engine/page_actions_test.go | 4 +- v2/pkg/templates/compile.go | 6 +- v2/pkg/tmplexec/doc.go | 5 + v2/pkg/tmplexec/exec.go | 117 +++++++++ .../flow}/builtin/dedupe.go | 0 v2/pkg/tmplexec/flow/doc.go | 1 + v2/pkg/tmplexec/flow/flow_executor.go | 237 ++++++++++++++++++ .../flow}/flow_executor_test.go | 2 +- .../flow/flow_internal.go} | 229 +---------------- v2/pkg/tmplexec/flow/options.go | 48 ++++ .../testcases/condition-flow-extractors.yaml | 0 .../condition-flow-no-operators.yaml | 0 .../flow}/testcases/condition-flow.yaml | 0 .../flow}/testcases/nuclei-flow-dns-id.yaml | 0 .../testcases/nuclei-flow-dns-prefix.yaml | 0 .../flow}/testcases/nuclei-flow-dns.yaml | 0 v2/pkg/tmplexec/flow/util.go | 23 ++ v2/pkg/tmplexec/generic/exec.go | 89 +++++++ v2/pkg/tmplexec/interface.go | 22 ++ 20 files changed, 555 insertions(+), 456 deletions(-) delete mode 100644 v2/pkg/protocols/common/executer/executer.go create mode 100644 v2/pkg/tmplexec/doc.go create mode 100644 v2/pkg/tmplexec/exec.go rename v2/pkg/{protocols/common/executer => tmplexec/flow}/builtin/dedupe.go (100%) create mode 100644 v2/pkg/tmplexec/flow/doc.go create mode 100644 v2/pkg/tmplexec/flow/flow_executor.go rename v2/pkg/{protocols/common/executer => tmplexec/flow}/flow_executor_test.go (99%) rename v2/pkg/{protocols/common/executer/flow_executor.go => tmplexec/flow/flow_internal.go} (53%) create mode 100644 v2/pkg/tmplexec/flow/options.go rename v2/pkg/{protocols/common/executer => tmplexec/flow}/testcases/condition-flow-extractors.yaml (100%) rename v2/pkg/{protocols/common/executer => tmplexec/flow}/testcases/condition-flow-no-operators.yaml (100%) rename v2/pkg/{protocols/common/executer => tmplexec/flow}/testcases/condition-flow.yaml (100%) rename v2/pkg/{protocols/common/executer => tmplexec/flow}/testcases/nuclei-flow-dns-id.yaml (100%) rename v2/pkg/{protocols/common/executer => tmplexec/flow}/testcases/nuclei-flow-dns-prefix.yaml (100%) rename v2/pkg/{protocols/common/executer => tmplexec/flow}/testcases/nuclei-flow-dns.yaml (100%) create mode 100644 v2/pkg/tmplexec/flow/util.go create mode 100644 v2/pkg/tmplexec/generic/exec.go create mode 100644 v2/pkg/tmplexec/interface.go diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go deleted file mode 100644 index 94905003bd..0000000000 --- a/v2/pkg/protocols/common/executer/executer.go +++ /dev/null @@ -1,228 +0,0 @@ -package executer - -import ( - "fmt" - "strings" - "sync/atomic" - - "github.com/pkg/errors" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" - errorutil "github.com/projectdiscovery/utils/errors" - mapsutil "github.com/projectdiscovery/utils/maps" -) - -// Executer executes a group of requests for a protocol -type Executer struct { - requests []protocols.Request - options *protocols.ExecutorOptions -} - -var _ protocols.Executer = &Executer{} - -// NewExecuter creates a new request executer for list of requests -func NewExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) *Executer { - return &Executer{requests: requests, options: options} -} - -// Compile compiles the execution generators preparing any requests possible. -func (e *Executer) Compile() error { - cliOptions := e.options.Options - - for _, request := range e.requests { - if err := request.Compile(e.options); err != nil { - var dslCompilationError *dsl.CompilationError - if errors.As(err, &dslCompilationError) { - if cliOptions.Verbose { - rawErrorMessage := dslCompilationError.Error() - formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "." - gologger.Warning().Msgf(formattedErrorMessage) - gologger.Info().Msgf("The available custom DSL functions are:") - fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor)) - } - } - return err - } - } - return nil -} - -// Requests returns the total number of requests the rule will perform -func (e *Executer) Requests() int { - var count int - for _, request := range e.requests { - count += request.Requests() - } - return count -} - -// Execute executes the protocol group and returns true or false if results were found. -func (e *Executer) Execute(input *contextargs.Context) (bool, error) { - results := &atomic.Bool{} - - var lastMatcherEvent *output.InternalWrappedEvent - writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { - if !results.Load() && matcherStatus { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } - results.CompareAndSwap(false, true) - } - } - - cliExecutorCallback := func(event *output.InternalWrappedEvent) { - // If no results were found, and also interactsh is not being used - // in that case we can skip it, otherwise we've to show failure in - // case of matcher-status flag. - if !event.HasOperatorResult() && !event.UsesInteractsh { - lastMatcherEvent = event - } else { - if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { - results.CompareAndSwap(false, true) - } else { - lastMatcherEvent = event - } - } - } - if e.options.Flow != "" { - return e.executeFlow(input, cliExecutorCallback) - } - _, err := e.executeWithCallback(input, results, cliExecutorCallback) - if lastMatcherEvent != nil { - writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) - } - return results.Load(), err -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (e *Executer) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { - gologger.Info().Msgf("[%s] Running on %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint()) - userCallback := func(event *output.InternalWrappedEvent) { - if event != nil { - callback(event) - } - } - var err error - if e.options.Flow != "" { - _, err = e.executeFlow(input, userCallback) - } else { - _, err = e.executeWithCallback(input, nil, userCallback) - } - return err -} - -// executeWithCallback executes the protocol requests and calls the callback for each result. -func (e *Executer) executeWithCallback(input *contextargs.Context, results *atomic.Bool, callback protocols.OutputEventCallback) (bool, error) { - if results == nil { - results = &atomic.Bool{} - } - dynamicValues := make(map[string]interface{}) - if input.HasArgs() { - input.ForEach(func(key string, value interface{}) { - dynamicValues[key] = value - }) - } - previous := make(map[string]interface{}) - - for _, req := range e.requests { - inputItem := input.Clone() - if e.options.InputHelper != nil && input.MetaInput.Input != "" { - if inputItem.MetaInput.Input = e.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" { - return false, nil - } - } - - err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { - if event == nil { - // ideally this should never happen since protocol exits on error and callback is not called - return - } - ID := req.GetID() - if ID != "" { - builder := &strings.Builder{} - for k, v := range event.InternalEvent { - builder.WriteString(ID) - builder.WriteString("_") - builder.WriteString(k) - previous[builder.String()] = v - builder.Reset() - } - } - if event.HasOperatorResult() { - results.CompareAndSwap(false, true) - } - // for ExecuteWithResults : this callback will execute user defined callback and some error handling - // for Execute : this callback will print the result to output - callback(event) - }) - if err != nil { - if e.options.HostErrorsCache != nil { - e.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) - } - gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint(), err) - } - // If a match was found and stop at first match is set, break out of the loop and return - if results.Load() && (e.options.StopAtFirstMatch || e.options.Options.StopAtFirstMatch) { - break - } - } - return results.Load(), nil -} - -// ExecuteFlow executes template as specified in js flow -// it is mutually exclusive with executeWithCallback -func (e *Executer) executeFlow(input *contextargs.Context, callback protocols.OutputEventCallback) (bool, error) { - allprotos := make(map[string][]protocols.Request) - for _, req := range e.requests { - if req.Type() == types.MultiProtocol { - // multiprotocol execution is also mutually exclusive with flow and is slightly advanced version of executeWithCallbac() - // if request type is multiprotocol , then array does not contain any other request type - return e.executeWithCallback(input, nil, nil) - } - switch req.Type() { - case types.DNSProtocol: - allprotos[types.DNSProtocol.String()] = append(allprotos[types.DNSProtocol.String()], req) - case types.HTTPProtocol: - allprotos[types.HTTPProtocol.String()] = append(allprotos[types.HTTPProtocol.String()], req) - case types.NetworkProtocol: - allprotos[types.NetworkProtocol.String()] = append(allprotos[types.NetworkProtocol.String()], req) - case types.FileProtocol: - allprotos[types.FileProtocol.String()] = append(allprotos[types.FileProtocol.String()], req) - case types.HeadlessProtocol: - allprotos[types.HeadlessProtocol.String()] = append(allprotos[types.HeadlessProtocol.String()], req) - case types.SSLProtocol: - allprotos[types.SSLProtocol.String()] = append(allprotos[types.SSLProtocol.String()], req) - case types.WebsocketProtocol: - allprotos[types.WebsocketProtocol.String()] = append(allprotos[types.WebsocketProtocol.String()], req) - case types.WHOISProtocol: - allprotos[types.WHOISProtocol.String()] = append(allprotos[types.WHOISProtocol.String()], req) - case types.CodeProtocol: - allprotos[types.CodeProtocol.String()] = append(allprotos[types.CodeProtocol.String()], req) - default: - gologger.Error().Msgf("invalid request type %s", req.Type().String()) - } - } - - flow := &FlowExecutor{ - allProtocols: allprotos, - input: input, - options: e.options, - allErrs: mapsutil.SyncLockMap[string, error]{ - ReadOnly: atomic.Bool{}, - Map: make(map[string]error), - }, - results: &atomic.Bool{}, - } - - if err := flow.Compile(callback); err != nil { - return false, errorutil.NewWithErr(err).Msgf("could not compile flow") - } - - return flow.Execute() -} diff --git a/v2/pkg/protocols/headless/engine/page_actions_test.go b/v2/pkg/protocols/headless/engine/page_actions_test.go index 3524b47d86..7930598d21 100644 --- a/v2/pkg/protocols/headless/engine/page_actions_test.go +++ b/v2/pkg/protocols/headless/engine/page_actions_test.go @@ -349,7 +349,7 @@ func TestActionGetResource(t *testing.T) { Nuclei Test Page - + ` @@ -360,7 +360,7 @@ func TestActionGetResource(t *testing.T) { testHeadlessSimpleResponse(t, response, actions, 20*time.Second, func(page *Page, err error, out map[string]string) { require.Nil(t, err, "could not run page actions") - require.Equal(t, len(out["src"]), 3159, "could not find resource") + require.Equal(t, len(out["src"]), 121808, "could not find resource") }) } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 964f5f3cf1..07b29fe17f 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -13,10 +13,10 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec" "github.com/projectdiscovery/nuclei/v2/pkg/utils" "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" @@ -161,7 +161,7 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsCode)...) } } - template.Executer = executer.NewExecuter(requests, &options) + template.Executer = tmplexec.NewTemplateExecuter(requests, &options) return nil } @@ -206,7 +206,7 @@ mainLoop: } if len(operatorsList) > 0 { options.Operators = operatorsList - template.Executer = executer.NewExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) + template.Executer = tmplexec.NewTemplateExecuter([]protocols.Request{&offlinehttp.Request{}}, &options) return nil } diff --git a/v2/pkg/tmplexec/doc.go b/v2/pkg/tmplexec/doc.go new file mode 100644 index 0000000000..88207b9ec9 --- /dev/null +++ b/v2/pkg/tmplexec/doc.go @@ -0,0 +1,5 @@ +package tmplexec + +// tmplexec is package that provides +// template executors it is one level higher than protocols +// and deals with execution of entire template diff --git a/v2/pkg/tmplexec/exec.go b/v2/pkg/tmplexec/exec.go new file mode 100644 index 0000000000..2f06b586cd --- /dev/null +++ b/v2/pkg/tmplexec/exec.go @@ -0,0 +1,117 @@ +package tmplexec + +import ( + "errors" + "fmt" + "strings" + "sync/atomic" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/operators/common/dsl" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/generic" +) + +// TemplateExecutor is an executor for a template +type TemplateExecuter struct { + requests []protocols.Request + options *protocols.ExecutorOptions + engine TemplateEngine + results *atomic.Bool +} + +// Both executer & Executor are correct spellings (its open to interpretation) + +var _ protocols.Executer = &TemplateExecuter{} + +// NewTemplateExecuter creates a new request TemplateExecuter for list of requests +func NewTemplateExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) *TemplateExecuter { + e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}} + if options.Flow != "" { + e.engine = flow.NewFlowExecutor(requests, options, e.results) + } else { + e.engine = generic.NewGenericEngine(requests, options, e.results) + } + return e +} + +// Compile compiles the execution generators preparing any requests possible. +func (e *TemplateExecuter) Compile() error { + cliOptions := e.options.Options + + for _, request := range e.requests { + if err := request.Compile(e.options); err != nil { + var dslCompilationError *dsl.CompilationError + if errors.As(err, &dslCompilationError) { + if cliOptions.Verbose { + rawErrorMessage := dslCompilationError.Error() + formattedErrorMessage := strings.ToUpper(rawErrorMessage[:1]) + rawErrorMessage[1:] + "." + gologger.Warning().Msgf(formattedErrorMessage) + gologger.Info().Msgf("The available custom DSL functions are:") + fmt.Println(dsl.GetPrintableDslFunctionSignatures(cliOptions.NoColor)) + } + } + return err + } + } + return e.engine.Compile() +} + +// Requests returns the total number of requests the rule will perform +func (e *TemplateExecuter) Requests() int { + var count int + for _, request := range e.requests { + count += request.Requests() + } + return count +} + +// Execute executes the protocol group and returns true or false if results were found. +func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { + results := &atomic.Bool{} + + var lastMatcherEvent *output.InternalWrappedEvent + writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { + if !results.Load() && matcherStatus { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + results.CompareAndSwap(false, true) + } + } + + cliExecutorCallback := func(event *output.InternalWrappedEvent) { + // If no results were found, and also interactsh is not being used + // in that case we can skip it, otherwise we've to show failure in + // case of matcher-status flag. + if !event.HasOperatorResult() && !event.UsesInteractsh { + lastMatcherEvent = event + } else { + if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { + results.CompareAndSwap(false, true) + } else { + lastMatcherEvent = event + } + } + } + err := e.engine.ExecuteWithResults(input, cliExecutorCallback) + if lastMatcherEvent != nil { + writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) + } + return results.Load(), err +} + +// ExecuteWithResults executes the protocol requests and returns results instead of writing them. +func (e *TemplateExecuter) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + gologger.Info().Msgf("[%s] Running on %s\n", e.options.TemplateID, input.MetaInput.PrettyPrint()) + userCallback := func(event *output.InternalWrappedEvent) { + if event != nil { + callback(event) + } + } + return e.engine.ExecuteWithResults(input, userCallback) +} diff --git a/v2/pkg/protocols/common/executer/builtin/dedupe.go b/v2/pkg/tmplexec/flow/builtin/dedupe.go similarity index 100% rename from v2/pkg/protocols/common/executer/builtin/dedupe.go rename to v2/pkg/tmplexec/flow/builtin/dedupe.go diff --git a/v2/pkg/tmplexec/flow/doc.go b/v2/pkg/tmplexec/flow/doc.go new file mode 100644 index 0000000000..42882657be --- /dev/null +++ b/v2/pkg/tmplexec/flow/doc.go @@ -0,0 +1 @@ +package flow diff --git a/v2/pkg/tmplexec/flow/flow_executor.go b/v2/pkg/tmplexec/flow/flow_executor.go new file mode 100644 index 0000000000..0a1178869e --- /dev/null +++ b/v2/pkg/tmplexec/flow/flow_executor.go @@ -0,0 +1,237 @@ +package flow + +import ( + "fmt" + "io" + "strconv" + "strings" + "sync" + "sync/atomic" + + "github.com/dop251/goja" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" + templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" + mapsutil "github.com/projectdiscovery/utils/maps" + "go.uber.org/multierr" +) + +var ( + // ErrInvalidRequestID is a request id error + ErrInvalidRequestID = errorutil.NewWithFmt("invalid request id '%s' provided") +) + +// FlowExecutor is a flow executor for executing a flow +type FlowExecutor struct { + input *contextargs.Context + options *protocols.ExecutorOptions + + // javascript runtime reference and compiled program + jsVM *goja.Runtime + program *goja.Program // compiled js program + + // protocol requests and their callback functions + allProtocols map[string][]protocols.Request + protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js + callback func(event *output.InternalWrappedEvent) // result event callback + + // logic related variables + wg sync.WaitGroup + results *atomic.Bool + allErrs mapsutil.SyncLockMap[string, error] +} + +// NewFlowExecutor creates a new flow executor from a list of requests +func NewFlowExecutor(requests []protocols.Request, options *protocols.ExecutorOptions, results *atomic.Bool) *FlowExecutor { + allprotos := make(map[string][]protocols.Request) + for _, req := range requests { + switch req.Type() { + case templateTypes.DNSProtocol: + allprotos[templateTypes.DNSProtocol.String()] = append(allprotos[templateTypes.DNSProtocol.String()], req) + case templateTypes.HTTPProtocol: + allprotos[templateTypes.HTTPProtocol.String()] = append(allprotos[templateTypes.HTTPProtocol.String()], req) + case templateTypes.NetworkProtocol: + allprotos[templateTypes.NetworkProtocol.String()] = append(allprotos[templateTypes.NetworkProtocol.String()], req) + case templateTypes.FileProtocol: + allprotos[templateTypes.FileProtocol.String()] = append(allprotos[templateTypes.FileProtocol.String()], req) + case templateTypes.HeadlessProtocol: + allprotos[templateTypes.HeadlessProtocol.String()] = append(allprotos[templateTypes.HeadlessProtocol.String()], req) + case templateTypes.SSLProtocol: + allprotos[templateTypes.SSLProtocol.String()] = append(allprotos[templateTypes.SSLProtocol.String()], req) + case templateTypes.WebsocketProtocol: + allprotos[templateTypes.WebsocketProtocol.String()] = append(allprotos[templateTypes.WebsocketProtocol.String()], req) + case templateTypes.WHOISProtocol: + allprotos[templateTypes.WHOISProtocol.String()] = append(allprotos[templateTypes.WHOISProtocol.String()], req) + case templateTypes.CodeProtocol: + allprotos[templateTypes.CodeProtocol.String()] = append(allprotos[templateTypes.CodeProtocol.String()], req) + default: + gologger.Error().Msgf("invalid request type %s", req.Type().String()) + } + } + return &FlowExecutor{ + allProtocols: allprotos, + options: options, + allErrs: mapsutil.SyncLockMap[string, error]{ + ReadOnly: atomic.Bool{}, + Map: make(map[string]error), + }, + protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{}, + results: results, + } +} + +// Init initializes the flow executor all dependencies +// this compiles and prepares for execution of a flow +// since it has dependencies on variables and etc it can't be done moved to templates package +func (f *FlowExecutor) Compile() error { + if f.results == nil { + f.results = new(atomic.Bool) + } + // store all dynamic variables and other variables here + if f.options.TemplateCtx == nil { + f.options.TemplateCtx = contextargs.New() + } + // create a new js vm/runtime + f.jsVM = goja.New() + + // load all variables and evaluate with existing data + variableMap := f.options.Variables.Evaluate(f.options.TemplateCtx.GetAll()) + // cli options + optionVars := generators.BuildPayloadFromOptions(f.options.Options) + // constants + constants := f.options.Constants + allVars := generators.MergeMaps(variableMap, constants, optionVars) + // we support loading variables from files in variables , cli options and constants + // try to load if files exist + for k, v := range allVars { + if str, ok := v.(string); ok && len(str) < 150 && fileutil.FileExists(str) { + if value, err := f.ReadDataFromFile(str); err == nil { + allVars[k] = value + } else { + gologger.Warning().Msgf("could not load file '%s' for variable '%s': %s", str, k, err) + } + } + } + f.options.TemplateCtx.Merge(allVars) // merge all variables into template context + + // ---- define callback functions/objects---- + f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} + // iterate over all protocols and generate callback functions for each protocol + for p, requests := range f.allProtocols { + // for each protocol build a requestMap with reqID and protocol request + reqMap := mapsutil.Map[string, protocols.Request]{} + counter := 0 + proto := strings.ToLower(p) // donot use loop variables in callback functions directly + for index := range requests { + request := f.allProtocols[proto][index] + if request.GetID() != "" { + // if id is present use it + reqMap[request.GetID()] = request + } + // fallback to using index as id + // always allow index as id as a fallback + reqMap[strconv.Itoa(counter)] = request + counter++ + } + // ---define hook that allows protocol/request execution from js----- + // --- this is the actual callback that is executed when function is invoked in js---- + f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value { + opts := &ProtoOptions{ + protoName: proto, + } + for _, v := range call.Arguments { + switch value := v.Export().(type) { + case map[string]interface{}: + opts.LoadOptions(value) + default: + opts.reqIDS = append(opts.reqIDS, types.ToString(value)) + } + } + // parallel execution of protocols + if opts.Async { + f.wg.Add(1) + go func() { + defer f.wg.Done() + f.requestExecutor(reqMap, opts) + }() + return f.jsVM.ToValue(true) + } + + return f.jsVM.ToValue(f.requestExecutor(reqMap, opts)) + } + } + + // register all functions in javascript runtime + return f.registerBuiltInFunctions() +} + +// ExecuteWithResults executes the flow and returns results +func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + f.callback = callback + f.input = input + // -----Load all types of variables----- + // add all input args to template context + if f.input != nil && f.input.HasArgs() { + f.input.ForEach(func(key string, value interface{}) { + f.options.TemplateCtx.Set(key, value) + }) + } + if f.callback == nil { + return fmt.Errorf("output callback cannot be nil") + } + + // pass flow and execute the js vm and handle errors + value, err := f.jsVM.RunProgram(f.program) + if err != nil { + return errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) + } + f.wg.Wait() + runtimeErr := f.GetRuntimeErrors() + if runtimeErr != nil { + return errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") + } + if value.Export() != nil { + f.results.Store(value.ToBoolean()) + } else { + f.results.Store(true) + } + return nil +} + +// GetRuntimeErrors returns all runtime errors (i.e errors from all protocol combined) +func (f *FlowExecutor) GetRuntimeErrors() error { + errs := []error{} + for proto, err := range f.allErrs.GetAll() { + errs = append(errs, errorutil.NewWithErr(err).Msgf("failed to execute %v protocol", proto)) + } + return multierr.Combine(errs...) +} + +// ReadDataFromFile reads data from file respecting sandbox options +func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { + values := []string{} + // load file respecting sandbox + reader, err := f.options.Options.LoadHelperFile(payload, f.options.TemplatePath, f.options.Catalog) + if err != nil { + return values, err + } + defer reader.Close() + bin, err := io.ReadAll(reader) + if err != nil { + return values, err + } + for _, line := range strings.Split(string(bin), "\n") { + line = strings.TrimSpace(line) + if line != "" { + values = append(values, line) + } + } + return values, nil +} diff --git a/v2/pkg/protocols/common/executer/flow_executor_test.go b/v2/pkg/tmplexec/flow/flow_executor_test.go similarity index 99% rename from v2/pkg/protocols/common/executer/flow_executor_test.go rename to v2/pkg/tmplexec/flow/flow_executor_test.go index e50b6cd2aa..03290af30a 100644 --- a/v2/pkg/protocols/common/executer/flow_executor_test.go +++ b/v2/pkg/tmplexec/flow/flow_executor_test.go @@ -1,4 +1,4 @@ -package executer_test +package flow_test import ( "context" diff --git a/v2/pkg/protocols/common/executer/flow_executor.go b/v2/pkg/tmplexec/flow/flow_internal.go similarity index 53% rename from v2/pkg/protocols/common/executer/flow_executor.go rename to v2/pkg/tmplexec/flow/flow_internal.go index 9cd4b8ab70..9bb04e2b9b 100644 --- a/v2/pkg/protocols/common/executer/flow_executor.go +++ b/v2/pkg/tmplexec/flow/flow_internal.go @@ -1,151 +1,21 @@ -package executer +package flow import ( - "io" "reflect" - "strconv" "strings" - "sync" "sync/atomic" "github.com/dop251/goja" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/executer/builtin" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" - "github.com/projectdiscovery/nuclei/v2/pkg/types" - errorutil "github.com/projectdiscovery/utils/errors" - fileutil "github.com/projectdiscovery/utils/file" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow/builtin" mapsutil "github.com/projectdiscovery/utils/maps" - "go.uber.org/multierr" ) -var ( - ErrInvalidRequestID = errorutil.NewWithFmt("invalid request id '%s' provided") -) - -type ProtoOptions struct { - Hide bool - Async bool - protoName string - reqIDS []string - callback func(result *output.InternalWrappedEvent) -} - -// LoadOptions loads the protocol options from a map -func (P *ProtoOptions) LoadOptions(m map[string]interface{}) { - P.Hide = GetBool(m["hide"]) - P.Async = GetBool(m["async"]) -} - -type FlowExecutor struct { - input *contextargs.Context - allProtocols map[string][]protocols.Request - options *protocols.ExecutorOptions - allErrs mapsutil.SyncLockMap[string, error] - results *atomic.Bool - jsVM *goja.Runtime - program *goja.Program - protoFunctions map[string]func(call goja.FunctionCall) goja.Value // reqFunctions contains functions that allow executing requests/protocols from js - wg sync.WaitGroup -} - -// Init initializes the flow executor all dependencies -// this compiles and prepares for execution of a flow -// since it has dependencies on variables and etc it can't be done moved to templates package -func (f *FlowExecutor) Compile(callback func(event *output.InternalWrappedEvent)) error { - if f.results == nil { - f.results = new(atomic.Bool) - } - // store all dynamic variables and other variables here - if f.options.TemplateCtx == nil { - f.options.TemplateCtx = contextargs.New() - } - // create a new js vm/runtime - f.jsVM = goja.New() - - // -----Load all types of variables----- - // add all input args to template context - if f.input.HasArgs() { - f.input.ForEach(func(key string, value interface{}) { - f.options.TemplateCtx.Set(key, value) - }) - } - // load all variables and evaluate with existing data - variableMap := f.options.Variables.Evaluate(f.options.TemplateCtx.GetAll()) - // cli options - optionVars := generators.BuildPayloadFromOptions(f.options.Options) - // constants - constants := f.options.Constants - allVars := generators.MergeMaps(variableMap, constants, optionVars) - // we support loading variables from files in variables , cli options and constants - // try to load if files exist - for k, v := range allVars { - if str, ok := v.(string); ok && len(str) < 150 && fileutil.FileExists(str) { - if value, err := f.ReadDataFromFile(str); err == nil { - allVars[k] = value - } else { - gologger.Warning().Msgf("could not load file '%s' for variable '%s': %s", str, k, err) - } - } - } - f.options.TemplateCtx.Merge(allVars) - - // ---- define callback functions/objects---- - f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} - // iterate over all protocols and generate callback functions for each protocol - for p, requests := range f.allProtocols { - // for each protocol build a requestMap with reqID and protocol request - reqMap := mapsutil.Map[string, protocols.Request]{} - counter := 0 - proto := strings.ToLower(p) // donot use loop variables in callback functions directly - for index := range requests { - request := f.allProtocols[proto][index] - if request.GetID() != "" { - // if id is present use it - reqMap[request.GetID()] = request - } - // fallback to using index as id - // always allow index as id as a fallback - reqMap[strconv.Itoa(counter)] = request - counter++ - } - // ---define hook that allows protocol/request execution from js----- - // --- this is the actual callback that is executed when function is invoked in js---- - f.protoFunctions[proto] = func(call goja.FunctionCall) goja.Value { - opts := &ProtoOptions{ - callback: callback, - protoName: proto, - } - for _, v := range call.Arguments { - switch value := v.Export().(type) { - case map[string]interface{}: - opts.LoadOptions(value) - default: - opts.reqIDS = append(opts.reqIDS, types.ToString(value)) - } - } - if opts.Async { - f.wg.Add(1) - go func() { - defer f.wg.Done() - f.requestExecutor(reqMap, opts) - }() - return f.jsVM.ToValue(true) - } - - return f.jsVM.ToValue(f.requestExecutor(reqMap, opts)) - } - } - - // register all built in functions - return f.RegisterBuiltInFunctions() -} +// contains all internal/unexported methods of flow // requestExecutor executes a protocol/request and returns true if any matcher was found func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool { @@ -164,7 +34,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req if result != nil { f.results.CompareAndSwap(false, true) if !opts.Hide { - opts.callback(result) + f.callback(result) } // export dynamic values from operators (i.e internal:true) // add add it to template context @@ -220,7 +90,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req if result != nil { f.results.CompareAndSwap(false, true) if !opts.Hide { - opts.callback(result) + f.callback(result) } // export dynamic values from operators (i.e internal:true) // add add it to template context @@ -246,8 +116,8 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req return matcherStatus.Load() } -// RegisterBuiltInFunctions registers all built in functions for the flow -func (f *FlowExecutor) RegisterBuiltInFunctions() error { +// registerBuiltInFunctions registers all built in functions for the flow +func (f *FlowExecutor) registerBuiltInFunctions() error { // currently we register following builtin functions // log -> log to stdout with [JS] prefix should only be used for debugging // set -> set a variable in template context @@ -375,88 +245,3 @@ func (f *FlowExecutor) RegisterBuiltInFunctions() error { f.program = program return nil } - -// Execute executes the flow -func (f *FlowExecutor) Execute() (bool, error) { - // pass flow and execute the js vm and handle errors - value, err := f.jsVM.RunProgram(f.program) - if err != nil { - return false, errorutil.NewWithErr(err).Msgf("failed to execute flow\n%v\n", f.options.Flow) - } - f.wg.Wait() - runtimeErr := f.GetRuntimeErrors() - if runtimeErr != nil { - return false, errorutil.NewWithErr(runtimeErr).Msgf("got following errors while executing flow") - } - if value.Export() != nil { - return value.ToBoolean(), nil - } - return f.results.Load(), nil -} - -func (f *FlowExecutor) GetRuntimeErrors() error { - errs := []error{} - for proto, err := range f.allErrs.GetAll() { - errs = append(errs, errorutil.NewWithErr(err).Msgf("failed to execute %v protocol", proto)) - } - return multierr.Combine(errs...) -} - -// ReadDataFromFile reads data from file respecting sandbox options -func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { - values := []string{} - // load file respecting sandbox - reader, err := f.options.Options.LoadHelperFile(payload, f.options.TemplatePath, f.options.Catalog) - if err != nil { - return values, err - } - defer reader.Close() - bin, err := io.ReadAll(reader) - if err != nil { - return values, err - } - for _, line := range strings.Split(string(bin), "\n") { - line = strings.TrimSpace(line) - if line != "" { - values = append(values, line) - } - } - return values, nil -} - -// Checks if template has matchers -func hasMatchers(all []*operators.Operators) bool { - for _, operator := range all { - if len(operator.Matchers) > 0 { - return true - } - } - return false -} - -// hasOperators checks if template has operators (i.e matchers/extractors) -func hasOperators(all []*operators.Operators) bool { - for _, operator := range all { - if operator != nil { - return true - } - } - return false -} - -// GetBool returns bool value from interface -func GetBool(value interface{}) bool { - if value == nil { - return false - } - switch v := value.(type) { - case bool: - return v - default: - tmpValue := types.ToString(value) - if strings.EqualFold(tmpValue, "true") { - return true - } - } - return false -} diff --git a/v2/pkg/tmplexec/flow/options.go b/v2/pkg/tmplexec/flow/options.go new file mode 100644 index 0000000000..3d845a13b3 --- /dev/null +++ b/v2/pkg/tmplexec/flow/options.go @@ -0,0 +1,48 @@ +package flow + +import ( + "strings" + + "github.com/projectdiscovery/nuclei/v2/pkg/types" +) + +// ProtoOptions are options that can be passed to flow protocol callback +// ex: dns(protoOptions) <- protoOptions are optional and can be anything +type ProtoOptions struct { + Hide bool + Async bool + protoName string + reqIDS []string +} + +// Examples +// dns() <- callback without any options +// dns(1) or dns(1,3) <- callback with index of protocol in template request at 1 or 1 and 3 +// dns("probe-http") or dns("extract-vpc","probe-http") <- callback with id's instead of index of request in template +// dns({hide:true}) or dns({hide:true,async:true}) <- callback with protocol options +// hide - hides result/event from output & sdk +// async - executes protocols in parallel (implicit wait no need to specify wait) +// Note: all of these options are optional and can be combined together in any order + +// LoadOptions loads the protocol options from a map +func (P *ProtoOptions) LoadOptions(m map[string]interface{}) { + P.Hide = GetBool(m["hide"]) + P.Async = GetBool(m["async"]) +} + +// GetBool returns bool value from interface +func GetBool(value interface{}) bool { + if value == nil { + return false + } + switch v := value.(type) { + case bool: + return v + default: + tmpValue := types.ToString(value) + if strings.EqualFold(tmpValue, "true") { + return true + } + } + return false +} diff --git a/v2/pkg/protocols/common/executer/testcases/condition-flow-extractors.yaml b/v2/pkg/tmplexec/flow/testcases/condition-flow-extractors.yaml similarity index 100% rename from v2/pkg/protocols/common/executer/testcases/condition-flow-extractors.yaml rename to v2/pkg/tmplexec/flow/testcases/condition-flow-extractors.yaml diff --git a/v2/pkg/protocols/common/executer/testcases/condition-flow-no-operators.yaml b/v2/pkg/tmplexec/flow/testcases/condition-flow-no-operators.yaml similarity index 100% rename from v2/pkg/protocols/common/executer/testcases/condition-flow-no-operators.yaml rename to v2/pkg/tmplexec/flow/testcases/condition-flow-no-operators.yaml diff --git a/v2/pkg/protocols/common/executer/testcases/condition-flow.yaml b/v2/pkg/tmplexec/flow/testcases/condition-flow.yaml similarity index 100% rename from v2/pkg/protocols/common/executer/testcases/condition-flow.yaml rename to v2/pkg/tmplexec/flow/testcases/condition-flow.yaml diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-id.yaml b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-id.yaml similarity index 100% rename from v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-id.yaml rename to v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-id.yaml diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml similarity index 100% rename from v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns-prefix.yaml rename to v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns-prefix.yaml diff --git a/v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml b/v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml similarity index 100% rename from v2/pkg/protocols/common/executer/testcases/nuclei-flow-dns.yaml rename to v2/pkg/tmplexec/flow/testcases/nuclei-flow-dns.yaml diff --git a/v2/pkg/tmplexec/flow/util.go b/v2/pkg/tmplexec/flow/util.go new file mode 100644 index 0000000000..6e1c343f88 --- /dev/null +++ b/v2/pkg/tmplexec/flow/util.go @@ -0,0 +1,23 @@ +package flow + +import "github.com/projectdiscovery/nuclei/v2/pkg/operators" + +// Checks if template has matchers +func hasMatchers(all []*operators.Operators) bool { + for _, operator := range all { + if len(operator.Matchers) > 0 { + return true + } + } + return false +} + +// hasOperators checks if template has operators (i.e matchers/extractors) +func hasOperators(all []*operators.Operators) bool { + for _, operator := range all { + if operator != nil { + return true + } + } + return false +} diff --git a/v2/pkg/tmplexec/generic/exec.go b/v2/pkg/tmplexec/generic/exec.go new file mode 100644 index 0000000000..f2900c2fa3 --- /dev/null +++ b/v2/pkg/tmplexec/generic/exec.go @@ -0,0 +1,89 @@ +package generic + +import ( + "strings" + "sync/atomic" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" +) + +// generic engine as name suggests is a generic template +// execution engine and executes all protocol requests one after another +// without any logic in between +type Generic struct { + requests []protocols.Request + options *protocols.ExecutorOptions + results *atomic.Bool +} + +// NewGenericEngine creates a new generic engine from a list of requests +func NewGenericEngine(requests []protocols.Request, options *protocols.ExecutorOptions, results *atomic.Bool) *Generic { + if results == nil { + results = &atomic.Bool{} + } + return &Generic{requests: requests, options: options, results: results} +} + +// Compile engine specific compilation +func (g *Generic) Compile() error { + // protocol/ request is already handled by template executer + return nil +} + +// ExecuteWithResults executes the template and returns results +func (g *Generic) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + dynamicValues := make(map[string]interface{}) + if input.HasArgs() { + input.ForEach(func(key string, value interface{}) { + dynamicValues[key] = value + }) + } + previous := make(map[string]interface{}) + + for _, req := range g.requests { + inputItem := input.Clone() + if g.options.InputHelper != nil && input.MetaInput.Input != "" { + if inputItem.MetaInput.Input = g.options.InputHelper.Transform(inputItem.MetaInput.Input, req.Type()); inputItem.MetaInput.Input == "" { + return nil + } + } + + err := req.ExecuteWithResults(inputItem, dynamicValues, previous, func(event *output.InternalWrappedEvent) { + if event == nil { + // ideally this should never happen since protocol exits on error and callback is not called + return + } + ID := req.GetID() + if ID != "" { + builder := &strings.Builder{} + for k, v := range event.InternalEvent { + builder.WriteString(ID) + builder.WriteString("_") + builder.WriteString(k) + previous[builder.String()] = v + builder.Reset() + } + } + if event.HasOperatorResult() { + g.results.CompareAndSwap(false, true) + } + // for ExecuteWithResults : this callback will execute user defined callback and some error handling + // for Execute : this callback will print the result to output + callback(event) + }) + if err != nil { + if g.options.HostErrorsCache != nil { + g.options.HostErrorsCache.MarkFailed(input.MetaInput.ID(), err) + } + gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, input.MetaInput.PrettyPrint(), err) + } + // If a match was found and stop at first match is set, break out of the loop and return + if g.results.Load() && (g.options.StopAtFirstMatch || g.options.Options.StopAtFirstMatch) { + break + } + } + return nil +} diff --git a/v2/pkg/tmplexec/interface.go b/v2/pkg/tmplexec/interface.go new file mode 100644 index 0000000000..076e8b5333 --- /dev/null +++ b/v2/pkg/tmplexec/interface.go @@ -0,0 +1,22 @@ +package tmplexec + +import ( + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" +) + +// TemplateEngine is a template executor with different functionality +// Ex: +// 1. generic => executes all protocol requests one after another (Done) +// 2. flow => executes protocol requests based on how they are defined in flow (Done) +// 3. cluster => clusters multiple requests from different templates and executes them thus reducing unneccessary requests (investigate && TODO) +// 4. multiprotocol => executes multiple protocols in parallel (TODO) +type TemplateEngine interface { + // Note: below methods only need to implement extra / engine specific functionality + // basic request compilation , callbacks , cli output callback etc are handled by `TemplateExecuter` and no need to do it again + // Extra Compilation (if any) + Compile() error + + // ExecuteWithResults executes the template and returns results + ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error +} From 2f39929aaf7b4ade9ff29e31c53dd81fabc24d98 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 22 Aug 2023 19:31:33 +0530 Subject: [PATCH 27/38] flow executor working --- v2/go.mod | 8 +- v2/go.sum | 8 + .../common/contextargs/contextargs.go | 2 +- v2/pkg/protocols/multi/doc.go | 6 - v2/pkg/protocols/multi/request.go | 172 --------------- v2/pkg/protocols/protocols.go | 7 +- v2/pkg/templates/compile.go | 11 +- v2/pkg/templates/compile_test.go | 14 -- v2/pkg/templates/templates.go | 53 +++-- v2/pkg/templates/types/types.go | 3 - v2/pkg/tmplexec/README.md | 11 + v2/pkg/tmplexec/exec.go | 46 +++- v2/pkg/tmplexec/flow/README.md | 196 ++++++++++++++++++ v2/pkg/tmplexec/flow/flow_executor.go | 19 +- v2/pkg/tmplexec/flow/flow_executor_test.go | 1 - v2/pkg/tmplexec/flow/flow_internal.go | 4 + v2/pkg/tmplexec/generic/exec.go | 7 +- v2/pkg/tmplexec/interface.go | 15 +- .../multi => tmplexec/multiproto}/README.md | 12 +- v2/pkg/tmplexec/multiproto/doc.go | 4 + v2/pkg/tmplexec/multiproto/multi.go | 109 ++++++++++ .../multiproto/multi_test.go} | 12 +- .../testcases/multiprotodynamic.yaml | 0 .../testcases/multiprotowithprefix.yaml | 0 24 files changed, 456 insertions(+), 264 deletions(-) delete mode 100644 v2/pkg/protocols/multi/doc.go delete mode 100644 v2/pkg/protocols/multi/request.go create mode 100644 v2/pkg/tmplexec/README.md create mode 100644 v2/pkg/tmplexec/flow/README.md rename v2/pkg/{protocols/multi => tmplexec/multiproto}/README.md (77%) create mode 100644 v2/pkg/tmplexec/multiproto/doc.go create mode 100644 v2/pkg/tmplexec/multiproto/multi.go rename v2/pkg/{protocols/multi/request_test.go => tmplexec/multiproto/multi_test.go} (86%) rename v2/pkg/{protocols/multi => tmplexec/multiproto}/testcases/multiprotodynamic.yaml (100%) rename v2/pkg/{protocols/multi => tmplexec/multiproto}/testcases/multiprotowithprefix.yaml (100%) diff --git a/v2/go.mod b/v2/go.mod index 748075ca14..34fc7753fe 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -41,7 +41,7 @@ require ( go.uber.org/multierr v1.11.0 golang.org/x/net v0.12.0 golang.org/x/oauth2 v0.10.0 - golang.org/x/text v0.11.0 + golang.org/x/text v0.12.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 ) @@ -60,7 +60,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 github.com/docker/go-units v0.5.0 - github.com/dop251/goja v0.0.0-20230707174833-636fdf960de1 + github.com/dop251/goja v0.0.0-20230812105242-81d76064690d github.com/fatih/structs v1.1.0 github.com/go-git/go-git/v5 v5.7.0 github.com/h2non/filetype v1.1.3 @@ -106,14 +106,14 @@ require ( github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect github.com/cloudflare/circl v1.3.3 // indirect - github.com/dlclark/regexp2 v1.8.1 // indirect + github.com/dlclark/regexp2 v1.10.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gaukas/godicttls v0.0.3 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/google/certificate-transparency-go v1.1.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect - github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect diff --git a/v2/go.sum b/v2/go.sum index 35786339c0..4eb1486264 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -152,12 +152,16 @@ github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwu github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja v0.0.0-20230707174833-636fdf960de1 h1:sC/DYk3eEi5cKkpJX1vl+CpAM138dmuW7rutje9Eo4E= github.com/dop251/goja v0.0.0-20230707174833-636fdf960de1/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= +github.com/dop251/goja v0.0.0-20230812105242-81d76064690d h1:9aaGwVf4q+kknu+mROAXUApJ1DoOwhE8dGj/XLBYzWg= +github.com/dop251/goja v0.0.0-20230812105242-81d76064690d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= @@ -248,6 +252,8 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -750,6 +756,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/v2/pkg/protocols/common/contextargs/contextargs.go b/v2/pkg/protocols/common/contextargs/contextargs.go index 9b34457c67..dd594f73d1 100644 --- a/v2/pkg/protocols/common/contextargs/contextargs.go +++ b/v2/pkg/protocols/common/contextargs/contextargs.go @@ -116,7 +116,7 @@ func (ctx *Context) HasArgs() bool { func (ctx *Context) Clone() *Context { newCtx := &Context{ MetaInput: ctx.MetaInput.Clone(), - args: ctx.args, + args: ctx.args.Clone(), CookieJar: ctx.CookieJar, } return newCtx diff --git a/v2/pkg/protocols/multi/doc.go b/v2/pkg/protocols/multi/doc.go deleted file mode 100644 index ec688892c2..0000000000 --- a/v2/pkg/protocols/multi/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -package multi - -// multi is a wrapper protocol Request that allows multiple protocols requests to be executed -// multi protocol is just a wrapper so it should/does not include any protocol specific code - - diff --git a/v2/pkg/protocols/multi/request.go b/v2/pkg/protocols/multi/request.go deleted file mode 100644 index 4f092c0f8f..0000000000 --- a/v2/pkg/protocols/multi/request.go +++ /dev/null @@ -1,172 +0,0 @@ -package multi - -import ( - "strconv" - - "github.com/projectdiscovery/nuclei/v2/pkg/model" - "github.com/projectdiscovery/nuclei/v2/pkg/operators" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/extractors" - "github.com/projectdiscovery/nuclei/v2/pkg/operators/matchers" - "github.com/projectdiscovery/nuclei/v2/pkg/output" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" - errorutil "github.com/projectdiscovery/utils/errors" -) - -var _ protocols.Request = &Request{} - -// refer doc.go for package description , limitations etc - -// Request contains a multi protocol request -type Request struct { - // description: | - // ID is the unique id for the template. - // - // #### Good IDs - // - // A good ID uniquely identifies what the requests in the template - // are doing. Let's say you have a template that identifies a git-config - // file on the webservers, a good name would be `git-config-exposure`. Another - // example name is `azure-apps-nxdomain-takeover`. - // examples: - // - name: ID Example - // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" json:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` - // description: | - // Info contains metadata information about the template. - // examples: - // - value: exampleInfoStructure - Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"` - - // Queue is queue of all protocols present in the template - Queue []protocols.Request `yaml:"-" json:"-"` - // request executor options - options *protocols.ExecutorOptions `yaml:"-" json:"-"` -} - -// getLastRequest returns the last request in the queue -func (r *Request) getLastRequest() protocols.Request { - if len(r.Queue) == 0 { - return nil - } - return r.Queue[len(r.Queue)-1] -} - -// Requests returns the total number of requests template will send -func (r *Request) Requests() int { - var count int - for _, protocol := range r.Queue { - count += protocol.Requests() - } - return count -} - -// Compile compiles the protocol request for further execution. -func (r *Request) Compile(executerOptions *protocols.ExecutorOptions) error { - r.options = executerOptions - r.options.TemplateCtx = contextargs.New() - r.options.ProtocolType = types.MultiProtocol - for _, protocol := range r.Queue { - if err := protocol.Compile(r.options); err != nil { - return errorutil.NewWithErr(err).Msgf("failed to compile protocol %s", protocol.Type()) - } - } - return nil -} - -// GetID returns the unique template ID -func (r *Request) GetID() string { - return r.ID -} - -// Match executes matcher on model and returns true or false (used for clustering if request supports clustering) -func (r *Request) Match(data map[string]interface{}, matcher *matchers.Matcher) (bool, []string) { - return protocols.MakeDefaultMatchFunc(data, matcher) -} - -// Extract performs extracting operation for an extractor on model and returns true or false (used for clustering if request supports clustering) -func (r *Request) Extract(data map[string]interface{}, matcher *extractors.Extractor) map[string]struct{} { - return protocols.MakeDefaultExtractFunc(data, matcher) -} - -// ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (r *Request) ExecuteWithResults(input *contextargs.Context, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { - var finalProtoEvent *output.InternalWrappedEvent - // callback to process results from all protocols - multiProtoCallback := func(event *output.InternalWrappedEvent) { - finalProtoEvent = event - // export dynamic values from operators (i.e internal:true) - if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 { - for k, v := range event.OperatorsResult.DynamicValues { - // TBD: iterate-all is only supported in `http` protocol - // we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context) - // currently if dynamic value array only contains one value we replace it with the value - if len(v) == 1 { - r.options.TemplateCtx.Set(k, v[0]) - } else { - // Note: if extracted value contains multiple values then they can be accessed by indexing - // ex: if values are dynamic = []string{"a","b","c"} then they are available as - // dynamic = "a" , dynamic1 = "b" , dynamic2 = "c" - // we intentionally omit first index for unknown situations (where no of extracted values are not known) - for i, val := range v { - if i == 0 { - r.options.TemplateCtx.Set(k, val) - } else { - r.options.TemplateCtx.Set(k+strconv.Itoa(i), val) - } - } - } - } - } - } - - // template context: contains values extracted using `internal` extractor from previous protocols - // these values are extracted from each protocol in queue and are passed to next protocol in queue - // instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows) - // this makes it possible to use multi protocol templates in workflows - // Note: internal extractor values take precedence over dynamicValues from workflows (i.e other templates in workflow) - - // execute all protocols in the queue - for _, req := range r.Queue { - err := req.ExecuteWithResults(input, dynamicValues, previous, multiProtoCallback) - // if error skip execution of next protocols - if err != nil { - return err - } - } - // Review: how to handle events of multiple protocols in a single template - // currently the outer callback is only executed once (for the last protocol in queue) - // due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/executer/executer.go#L150 - // this causes addition of duplicated / unncessary variables with prefix template_id_all_variables - callback(finalProtoEvent) - - return nil -} - -// MakeResultEventItem creates a result event from internal wrapped event. Intended to be used by MakeResultEventItem internally -func (r *Request) MakeResultEventItem(wrapped *output.InternalWrappedEvent) *output.ResultEvent { - if r.getLastRequest() == nil { - return nil - } - return r.getLastRequest().MakeResultEventItem(wrapped) -} - -// MakeResultEvent creates a flat list of result events from an internal wrapped event, based on successful matchers and extracted data -func (r *Request) MakeResultEvent(wrapped *output.InternalWrappedEvent) []*output.ResultEvent { - return protocols.MakeDefaultResultEvent(r.getLastRequest(), wrapped) -} - -// GetCompiledOperators returns a list of the compiled operators -func (r *Request) GetCompiledOperators() []*operators.Operators { - last := r.getLastRequest() - if last == nil { - return nil - } - return last.GetCompiledOperators() -} - -// Type returns the type of the protocol request -func (r *Request) Type() types.ProtocolType { - return types.MultiProtocol -} diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 0d65e36ded..8886de16a5 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -93,6 +93,9 @@ type ExecutorOptions struct { ProtocolType templateTypes.ProtocolType // Flow is execution flow for the template (written in javascript) Flow string + // IsMultiProtocol is true if template has more than one protocol + IsMultiProtocol bool + // TemplateCtx retains state of template variables } // AddTemplateVars adds vars to template context with given template type as prefix @@ -101,7 +104,7 @@ func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolTyp // if we wan't to disable adding response variables and other variables to template context // this is the statement that does it . template context is currently only enabled for // multiprotocol and flow templates - if e.ProtocolType != templateTypes.MultiProtocol && e.Flow == "" { + if !e.IsMultiProtocol && e.Flow == "" { // no-op if not multi protocol template or flow template return } @@ -120,7 +123,7 @@ func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolTyp // AddTemplateVar adds given var to template context with given template type as prefix // this method is no-op if template is not multi protocol func (e *ExecutorOptions) AddTemplateVar(templateType templateTypes.ProtocolType, reqID string, key string, value interface{}) { - if e.ProtocolType != templateTypes.MultiProtocol && e.Flow == "" { + if !e.IsMultiProtocol && e.Flow == "" { // no-op if not multi protocol template or flow template return } diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index 07b29fe17f..a67a93c103 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -128,10 +128,13 @@ func (template *Template) compileProtocolRequests(options protocols.ExecutorOpti var requests []protocols.Request - if len(template.MultiProtoRequest.Queue) > 0 { - template.MultiProtoRequest.ID = template.ID - template.MultiProtoRequest.Info = template.Info - requests = append(requests, &template.MultiProtoRequest) + if template.hasMultipleRequests() { + // when multiple requests are present preserve the order of requests and protocols + // which is already done during unmarshalling + requests = template.RequestsQueue + if options.Flow == "" { + options.IsMultiProtocol = true + } } else { if len(template.RequestsDNS) > 0 { requests = append(requests, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) diff --git a/v2/pkg/templates/compile_test.go b/v2/pkg/templates/compile_test.go index 091dc07a9f..4e67918e50 100644 --- a/v2/pkg/templates/compile_test.go +++ b/v2/pkg/templates/compile_test.go @@ -25,7 +25,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/variables" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" "github.com/projectdiscovery/nuclei/v2/pkg/templates" - "github.com/projectdiscovery/nuclei/v2/pkg/templates/types" "github.com/projectdiscovery/nuclei/v2/pkg/testutils" "github.com/projectdiscovery/nuclei/v2/pkg/workflows" "github.com/projectdiscovery/ratelimit" @@ -197,16 +196,3 @@ func Test_WrongTemplate(t *testing.T) { require.Nil(t, got, "could not parse template") require.ErrorContains(t, err, "no requests defined ") } - -func Test_Multiprotocol(t *testing.T) { - setup() - got, err := templates.Parse("tests/multiproto.yaml", nil, executerOpts) - require.Nil(t, err, "could not parse template") - require.Equal(t, 3, got.Requests()) - require.Equal(t, types.MultiProtocol, got.Type()) - - got, err = templates.Parse("tests/multiproto.json", nil, executerOpts) - require.Nil(t, err, "could not parse template") - require.Equal(t, 3, got.Requests()) - require.Equal(t, types.MultiProtocol, got.Type()) -} diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index 14d14bd158..baf147d97b 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -14,7 +14,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/file" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/headless" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/http" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/multi" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/network" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/ssl" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/websocket" @@ -147,15 +146,13 @@ type Template struct { // Verified defines if the template signature is digitally verified Verified bool `yaml:"-" json:"-"` - // MultiProtoRequest (Internal) contains multi protocol request if multiple protocols are used - MultiProtoRequest multi.Request `yaml:"-" json:"-"` + // RequestsQueue contains all template requests in order (both protocol & request order) + RequestsQueue []protocols.Request `yaml:"-" json:"-"` } // Type returns the type of the template func (template *Template) Type() types.ProtocolType { switch { - case len(template.MultiProtoRequest.Queue) > 0: - return types.MultiProtocol case len(template.RequestsDNS) > 0: return types.DNSProtocol case len(template.RequestsFile) > 0: @@ -293,8 +290,9 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error if err != nil { return err } - // check if the template contains a multi protocols - if template.isMultiProtocol() { + // check if the template contains more than 1 protocol request + // if so preserve the order of the protocols and requests + if template.hasMultipleRequests() { var tempmap yaml.MapSlice err = unmarshal(&tempmap) if err != nil { @@ -309,44 +307,40 @@ func (template *Template) UnmarshalYAML(unmarshal func(interface{}) error) error arr = append(arr, key) } // add protocols to the protocol stack (the idea is to preserve the order of the protocols) - template.addProtocolsToQueue(arr...) + template.addRequestsToQueue(arr...) } return nil } -// Internal function to create a protocol stack from a template if the template is a multi protocol template -func (template *Template) addProtocolsToQueue(keys ...string) { +// addProtocolsToQueue adds protocol requests to the queue and preserves order of the protocols and requests +func (template *Template) addRequestsToQueue(keys ...string) { for _, key := range keys { switch key { case types.DNSProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsDNS)...) case types.FileProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsFile)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsFile)...) case types.HTTPProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) case types.HeadlessProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHeadless)...) case types.NetworkProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) case types.SSLProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsSSL)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsSSL)...) case types.WebsocketProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsWebsocket)...) case types.WHOISProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) case types.CodeProtocol.String(): - template.MultiProtoRequest.Queue = append(template.MultiProtoRequest.Queue, template.convertRequestToProtocolsRequest(template.RequestsCode)...) + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsCode)...) } } } -// isMultiProtocol checks if the template is a multi protocol template -func (template *Template) isMultiProtocol() bool { - // Review: if template contains more than 1 protocol request (doesn't matter if it is same protocol or different) - // it is considered as multi protocol template - if template.Flow != "" { - return false - } +// hasMultipleRequests checks if the template has multiple requests +// if so it preserves the order of the request during compile and execution +func (template *Template) hasMultipleRequests() bool { counter := len(template.RequestsDNS) + len(template.RequestsFile) + len(template.RequestsHTTP) + len(template.RequestsHeadless) + len(template.RequestsNetwork) + len(template.RequestsSSL) + @@ -374,8 +368,9 @@ func (template *Template) UnmarshalJSON(data []byte) error { if err != nil { return err } - // check if template contains multiple protocols - if template.isMultiProtocol() { + // check if the template contains more than 1 protocol request + // if so preserve the order of the protocols and requests + if template.hasMultipleRequests() { var tempMap map[string]interface{} err = json.Unmarshal(data, &tempMap) if err != nil { @@ -385,7 +380,7 @@ func (template *Template) UnmarshalJSON(data []byte) error { for k := range tempMap { arr = append(arr, k) } - template.addProtocolsToQueue(arr...) + template.addRequestsToQueue(arr...) } return nil } diff --git a/v2/pkg/templates/types/types.go b/v2/pkg/templates/types/types.go index b4f392f062..8fb0315777 100644 --- a/v2/pkg/templates/types/types.go +++ b/v2/pkg/templates/types/types.go @@ -38,8 +38,6 @@ const ( WHOISProtocol // name:code CodeProtocol - // name: multi - MultiProtocol limit InvalidProtocol ) @@ -57,7 +55,6 @@ var protocolMappings = map[ProtocolType]string{ WebsocketProtocol: "websocket", WHOISProtocol: "whois", CodeProtocol: "code", - MultiProtocol: "multi", } func GetSupportedProtocolTypes() ProtocolTypes { diff --git a/v2/pkg/tmplexec/README.md b/v2/pkg/tmplexec/README.md new file mode 100644 index 0000000000..26e80daf0a --- /dev/null +++ b/v2/pkg/tmplexec/README.md @@ -0,0 +1,11 @@ +# tmplexec + +tmplexec also known as template executer executes template it is different from `protocols` package which only contains logic within the scope of one protocol. tmplexec is resposible for executing `Template` with defined logic. with introduction of `multi protocol` and `flow` templates (deprecated package protocols/common/executer) did not seem appropriate/helpful anymore as it is outside of protocol scope and deals with execution of template which can contain 1 requests , or multiple requests of same protocol or multiple requests of different protocols. tmplexec is responsible for executing template and handling all logic related to it. + +## Engine/Backends + +Currently there are 3 engines for template execution + +- `Generic` => executes request[s] of same/one protocol +- `MultiProtocol` => executes requests of multiple protocols with shared logic between protocol requests see [multiprotocol](multiproto/README.md) +- `Flow` => executes requests of one or multiple protocol requests as specified by template in javascript (aka flow) [flow](flow/README.md) \ No newline at end of file diff --git a/v2/pkg/tmplexec/exec.go b/v2/pkg/tmplexec/exec.go index 2f06b586cd..7371d163a7 100644 --- a/v2/pkg/tmplexec/exec.go +++ b/v2/pkg/tmplexec/exec.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/writer" "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow" "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/generic" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/multiproto" ) // TemplateExecutor is an executor for a template @@ -30,12 +31,30 @@ var _ protocols.Executer = &TemplateExecuter{} // NewTemplateExecuter creates a new request TemplateExecuter for list of requests func NewTemplateExecuter(requests []protocols.Request, options *protocols.ExecutorOptions) *TemplateExecuter { + isMultiProto := false + lastProto := "" + for _, request := range requests { + if request.Type().String() != lastProto && lastProto != "" { + isMultiProto = true + break + } + lastProto = request.Type().String() + } + e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}} if options.Flow != "" { e.engine = flow.NewFlowExecutor(requests, options, e.results) } else { - e.engine = generic.NewGenericEngine(requests, options, e.results) + // Review: + // multiproto engine is only used if there is more than one protocol in template + // else we use generic engine (should we use multiproto engine for single protocol with multiple requests as well ?) + if isMultiProto { + e.engine = multiproto.NewMultiProtocol(requests, options, e.results) + } else { + e.engine = generic.NewGenericEngine(requests, options, e.results) + } } + return e } @@ -85,6 +104,10 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { } cliExecutorCallback := func(event *output.InternalWrappedEvent) { + if event == nil { + // something went wrong + return + } // If no results were found, and also interactsh is not being used // in that case we can skip it, otherwise we've to show failure in // case of matcher-status flag. @@ -98,7 +121,26 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { } } } - err := e.engine.ExecuteWithResults(input, cliExecutorCallback) + var err error + + // Note: this is required for flow executor + // flow executer is tightly coupled with lot of executor options + // and map , wg and other types earlier we tried to use (compile once and run multiple times) + // but it is causing lot of panic and nil pointer dereference issues + // so in compile step earlier we compile it to validate javascript syntax and other things + // and while executing we create new instance of flow executor everytime + if e.options.Flow != "" { + // compile flow + flowexec := flow.NewFlowExecutor(e.requests, e.options, results) + if err := flowexec.Compile(); err != nil { + return false, err + } + // execute flow + err = flowexec.ExecuteWithResults(input, cliExecutorCallback) + } else { + err = e.engine.ExecuteWithResults(input, cliExecutorCallback) + } + if lastMatcherEvent != nil { writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) } diff --git a/v2/pkg/tmplexec/flow/README.md b/v2/pkg/tmplexec/flow/README.md new file mode 100644 index 0000000000..ba7d1ccdb2 --- /dev/null +++ b/v2/pkg/tmplexec/flow/README.md @@ -0,0 +1,196 @@ +# flow + +flow is a new template engine/backend introduced in v3 which primarily adds 2 most awaited features +- conditional execution of requests (ex: `flow: dns() && http()`) +- request execution orchestration (iterate over a slice, request execution order) + +both of these features are implemented using javascript (ECMAScript 5.1) and are executed using [goja](https://github.com/dop251/goja) and we can say that flow adds support for request orchestration and conditional execution of requests in templates. + +## conditional execution + +Many times when writing complex templates we might need to add some extra checks (or conditional statements) before executing certain part of request + +A idea example of this would be when bruteforcing wordpress login with default usernames and passwords. we can write a new nuclei template for this it would be something like this +```yaml +id: wordpress-bruteforce +info: + name: WordPress Login Bruteforce + author: pdteam + severity: high + +http: + - method: POST + path: + - "{{BaseURL}}/wp-login.php" + payloads: + username: + - admin + - guest + - testuser + password: + - password123 + - qwertyuiop + - letmein + body: "log=§username§&pwd=§password§&wp-submit=Log+In" + attack: clusterbomb + matchers: + - type: word + words: + - "ERROR" + part: body + negative: true +``` + +but if we rethink this template, we can see that we are sending 9 requests without even checking first if the url actually exists or target site is actually a wordpress site. before v3 it was possible to do this by adding a extractor and sending additional content in say url fragment and it would fail if request was not successful and another way would be writing a workflow (2 templates and 1 workflow file i.e total 3 files for 1 template) but this is `hacky` and not a good solution. + +With flow in Nuclei v3 we can re-write this template to first check if target is a wordpress site and then execute bruteforce requests. +this can be achieved by doing as simple as `flow: http("check-wp") && http("bruteforce")` + +```yaml +id: wordpress-bruteforce +info: + name: WordPress Login Bruteforce + author: pdteam + severity: high + +flow: http("check-wp") && http("bruteforce") + +http: + - id: check-wp + method: GET + path: + - "{{BaseURL}}/wp-login.php" + + matchers: + - type: word + words: + - "WordPress" + part: body + - type: word + words: + - "wp-content" + part: body + matchers-condition: and + + - id: bruteforce + method: POST + path: + - "{{BaseURL}}/wp-login.php" + payloads: + username: + - admin + - guest + - testuser + password: + - password123 + - qwertyuiop + - letmein + body: "log=§username§&pwd=§password§&wp-submit=Log+In" + attack: clusterbomb + matchers: + - type: word + words: + - "ERROR" + part: body + negative: true +``` +**Note:** this is just a example template with poor matchers. refer 'nuclei-templates' repo for final template + +Now we can see the template is straight forward and easy to understand. we are first checking if target is a wordpress site and then executing bruteforce requests. This is just a simple example of conditional execution and flow accepts any Javascript (ECMAScript 5.1) expression/code so you are free to craft any conditional execution logic you want using for , if and whatnot. + +## request execution orchestration + +`conditional execution` is one simple use case of flow but `flow` is much more powerful than that for example it can be used to +- iterate over a slice of values and execute requests for each value (ex: [dns-flow-probe](testcases/nuclei-flow-dns.yaml)) +- extract values from one request and iterate over them and execute requests for each value (ex: [[dns-flow-probe](testcases/nuclei-flow-dns.yaml)](https://github.com/projectdiscovery/nuclei/blob/64098b6567a2d6b7fc3e376d61af73836b3277bb/integration_tests/flow/iterate-values-flow.yaml)) +- get/set values from/to template context (global variables) +- print/log values to stdout at xyz condition or while debugging +- adding custom logic during template execution (ex: if status code is 403 then login and then re-run it) +- use any/all ECMAScript 5.1 javascript (like objects,arrays etc) and build/transform variables/input at runtime +- update variables at runtime (ex: when jwt expires update it by using refresh token and then continue execution) +- and a lot more (this is just a tip of iceberg) + +orchestration can be understood as nuclei logic bindings for javascript (i.e two way interaction between javascript and nuclei for a specific template) + +To better understand orchestration we can try to build a template for vhost enumeration using flow. which usually requires writing / using a new tool +for simple vhost enumeration we need to +- do a PTR lookup for given ip +- get SSL ceritificate for given ip (i.e tls-grab) + - extract subject_cn from certificate + - extract subject_alt_names(SAN) from certificate + - filter out wildcard prefix from above values +- and finally bruteforce all found vhosts + + +**Now if we try to implement this in template it would something like this** +```yaml +# send a ssl request to get certificate +ssl: + - address: "{{Host}}:{{Port}}" + +# do a PTR lookup for given ip and get PTR value +dns: + - name: "{{FQDN}}" + type: PTR + + matchers: + - type: word + words: + - "IN\tPTR" + + extractors: + - type: regex + name: ptrValue + internal: true + group: 1 + regex: + - "IN\tPTR\t(.+)" + +# bruteforce all found vhosts +http: + - raw: + - | + GET / HTTP/1.1 + Host: {{vhost}} + + matchers: + - type: status + negative: true + status: + - 400 + - 502 + + extractors: + - type: dsl + dsl: + - '"VHOST: " + vhost + ", SC: " + status_code + ", CL: " + content_length' tarun@macbook:~/Codebase/nuclei/integration_tes +``` +**But this template is not yet ready as it is missing core logic i.e how we use all these obtained data and do bruteforce** +and this is where flow comes into picture. it is javascript code with two way bindings to nuclei. if we write javascript code in very simple terms it would be something like this +```javascript + ssl(); + dns(); + for (let vhost of iterate(template["ssl_subject_cn"],template["ssl_subject_an"])) { + set("vhost", vhost); + http(); } +``` + +With just extra 5 lines of javascript code we can achieve vhost enumeration and run it on scale while also handling filtering of wildcard prefix from vhost values. +In above Js code we are using some Nuclei functions and 1 Map lets understand what they do + +- `ssl()` => execute ssl request +- `dns()` => execute dns request +- `template["ssl_subject_cn"]` => get value of `ssl_subject_cn` from template context (global variables) +- `iterate()` => this is a nuclei helper function which iterates any type of value (array,map,string,number) while handling empty / nil values +- `set("vhost",vhost)` => creates new variable `vhost` in template and assigns value of `vhost` to it +- `http()` => execute http request + +For such complex use case of vhost enumeration just adding 5 lines of js code using nuclei helper functions we achieved vhost enumeration. + +**Is this template ready?** +No, we are still missing one thing i.e subject_cn can contain values like `*.projectdiscovery.io` and we need to remove the prefix `*.` +there are lot of ways to do this we can either +- use javscript `replace()` function to remove prefix (ex: `vhost.replace("*.","")`) +- use nuclei js helper function `trimLeft()` (ex: `trimLeft(vhost,"*.")`) +- use dsl helper functions in http request + diff --git a/v2/pkg/tmplexec/flow/flow_executor.go b/v2/pkg/tmplexec/flow/flow_executor.go index 0a1178869e..c551856aea 100644 --- a/v2/pkg/tmplexec/flow/flow_executor.go +++ b/v2/pkg/tmplexec/flow/flow_executor.go @@ -95,9 +95,7 @@ func (f *FlowExecutor) Compile() error { f.results = new(atomic.Bool) } // store all dynamic variables and other variables here - if f.options.TemplateCtx == nil { - f.options.TemplateCtx = contextargs.New() - } + f.options.TemplateCtx = contextargs.New() // create a new js vm/runtime f.jsVM = goja.New() @@ -167,13 +165,18 @@ func (f *FlowExecutor) Compile() error { return f.jsVM.ToValue(f.requestExecutor(reqMap, opts)) } } - - // register all functions in javascript runtime return f.registerBuiltInFunctions() } // ExecuteWithResults executes the flow and returns results func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + defer func() { + if e := recover(); e != nil { + gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", input.MetaInput.Input, e) + panic(e) + } + }() + f.callback = callback f.input = input // -----Load all types of variables----- @@ -186,7 +189,6 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p if f.callback == nil { return fmt.Errorf("output callback cannot be nil") } - // pass flow and execute the js vm and handle errors value, err := f.jsVM.RunProgram(f.program) if err != nil { @@ -235,3 +237,8 @@ func (f *FlowExecutor) ReadDataFromFile(payload string) ([]string, error) { } return values, nil } + +// Name returns the type of engine +func (f *FlowExecutor) Name() string { + return "flow" +} diff --git a/v2/pkg/tmplexec/flow/flow_executor_test.go b/v2/pkg/tmplexec/flow/flow_executor_test.go index 03290af30a..b946cb781c 100644 --- a/v2/pkg/tmplexec/flow/flow_executor_test.go +++ b/v2/pkg/tmplexec/flow/flow_executor_test.go @@ -56,7 +56,6 @@ func TestFlowTemplateWithIndex(t *testing.T) { gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("hackerone.com")) require.Nil(t, err, "could not execute template") require.True(t, gotresults) - // apart from parse->compile->execution this testcase checks if dynamic extracted variables are available value, ok := Template.Options.TemplateCtx.Get("nameservers") require.True(t, ok) diff --git a/v2/pkg/tmplexec/flow/flow_internal.go b/v2/pkg/tmplexec/flow/flow_internal.go index 9bb04e2b9b..ad594adb94 100644 --- a/v2/pkg/tmplexec/flow/flow_internal.go +++ b/v2/pkg/tmplexec/flow/flow_internal.go @@ -20,6 +20,10 @@ import ( // requestExecutor executes a protocol/request and returns true if any matcher was found func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool { defer func() { + // evaluate all variables after execution of each protocol + variableMap := f.options.Variables.Evaluate(f.options.TemplateCtx.GetAll()) + f.options.TemplateCtx.Merge(variableMap) // merge all variables into template context + // to avoid polling update template variables everytime we execute a protocol var m map[string]interface{} = f.options.TemplateCtx.GetAll() _ = f.jsVM.Set("template", m) diff --git a/v2/pkg/tmplexec/generic/exec.go b/v2/pkg/tmplexec/generic/exec.go index f2900c2fa3..c4c0af7509 100644 --- a/v2/pkg/tmplexec/generic/exec.go +++ b/v2/pkg/tmplexec/generic/exec.go @@ -11,7 +11,7 @@ import ( ) // generic engine as name suggests is a generic template -// execution engine and executes all protocol requests one after another +// execution engine and executes all requests one after another // without any logic in between type Generic struct { requests []protocols.Request @@ -87,3 +87,8 @@ func (g *Generic) ExecuteWithResults(input *contextargs.Context, callback protoc } return nil } + +// Type returns the type of engine +func (g *Generic) Name() string { + return "generic" +} diff --git a/v2/pkg/tmplexec/interface.go b/v2/pkg/tmplexec/interface.go index 076e8b5333..ec52f915e0 100644 --- a/v2/pkg/tmplexec/interface.go +++ b/v2/pkg/tmplexec/interface.go @@ -3,14 +3,22 @@ package tmplexec import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/generic" + "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/multiproto" +) + +var ( + _ TemplateEngine = &generic.Generic{} + _ TemplateEngine = &flow.FlowExecutor{} + _ TemplateEngine = &multiproto.MultiProtocol{} ) // TemplateEngine is a template executor with different functionality // Ex: // 1. generic => executes all protocol requests one after another (Done) // 2. flow => executes protocol requests based on how they are defined in flow (Done) -// 3. cluster => clusters multiple requests from different templates and executes them thus reducing unneccessary requests (investigate && TODO) -// 4. multiprotocol => executes multiple protocols in parallel (TODO) +// 3. multiprotocol => executes multiple protocols in parallel (Done) type TemplateEngine interface { // Note: below methods only need to implement extra / engine specific functionality // basic request compilation , callbacks , cli output callback etc are handled by `TemplateExecuter` and no need to do it again @@ -19,4 +27,7 @@ type TemplateEngine interface { // ExecuteWithResults executes the template and returns results ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error + + // Name returns name of template engine + Name() string } diff --git a/v2/pkg/protocols/multi/README.md b/v2/pkg/tmplexec/multiproto/README.md similarity index 77% rename from v2/pkg/protocols/multi/README.md rename to v2/pkg/tmplexec/multiproto/README.md index f905554001..09fc549fc1 100644 --- a/v2/pkg/protocols/multi/README.md +++ b/v2/pkg/tmplexec/multiproto/README.md @@ -1,9 +1,8 @@ ## multi protocol execution ### Implementation -when template is unmarshalled, if it uses more than one protocol, it will be converted to a multi protocol -and the order of the protocols will be preserved as they were in the template and are stored in Request.Queue -when template is compiled , we iterate over queue and compile all the requests in the queue +when template is unmarshalled, if it uses more than one protocol, then order of protocols is preserved and is same is passed to Executor +multiproto is engine/backend for TemplateExecutor which takes care of sharing logic between protocols and executing them in order ### Execution when multi protocol template is executed , all protocol requests present in Queue are executed in order @@ -13,13 +12,6 @@ and dynamic values extracted are added to template context. apart from extracted `internal:true` values response fields/values of protocol are added to template context at `ExecutorOptions.TemplateCtx` which takes care of sync and other issues if any. all response fields are prefixed with template type prefix ex: `ssl_subject_dn` -### Other Methods -Such templates are usually used when a particular vulnerability requires more than one protocol to be executed -and in such cases the final result is core of the logic hence all methods such as -Ex: MakeResultEventItem, MakeResultEvent, GetCompiledOperators -are not implemented in multi protocol and just call the same method on last protocol in queue - - ### Adding New Protocol to multi protocol execution logic while logic/implementation of multi protocol execution is abstracted. it requires 3 statements to be added in newly implemented protocol to make response fields of that protocol available to global context diff --git a/v2/pkg/tmplexec/multiproto/doc.go b/v2/pkg/tmplexec/multiproto/doc.go new file mode 100644 index 0000000000..a22460da36 --- /dev/null +++ b/v2/pkg/tmplexec/multiproto/doc.go @@ -0,0 +1,4 @@ +package multiproto + +// multiproto is a template executer engine that executes multiple protocols +// with shared logic in between diff --git a/v2/pkg/tmplexec/multiproto/multi.go b/v2/pkg/tmplexec/multiproto/multi.go new file mode 100644 index 0000000000..68d2eb9bc4 --- /dev/null +++ b/v2/pkg/tmplexec/multiproto/multi.go @@ -0,0 +1,109 @@ +package multiproto + +import ( + "strconv" + "sync/atomic" + + "github.com/projectdiscovery/nuclei/v2/pkg/output" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/generators" +) + +// Mutliprotocol is a template executer engine that executes multiple protocols +// with logic in between +type MultiProtocol struct { + requests []protocols.Request + options *protocols.ExecutorOptions + results *atomic.Bool +} + +// NewMultiProtocol creates a new multiprotocol template engine from a list of requests +func NewMultiProtocol(requests []protocols.Request, options *protocols.ExecutorOptions, results *atomic.Bool) *MultiProtocol { + if results == nil { + results = &atomic.Bool{} + } + return &MultiProtocol{requests: requests, options: options, results: results} +} + +// Compile engine specific compilation +func (m *MultiProtocol) Compile() error { + // protocol/ request is already handled by template executer + m.options.TemplateCtx = contextargs.New() + // load all variables and evaluate with existing data + variableMap := m.options.Variables.Evaluate(m.options.TemplateCtx.GetAll()) + // cli options + optionVars := generators.BuildPayloadFromOptions(m.options.Options) + // constants + constants := m.options.Constants + allVars := generators.MergeMaps(variableMap, constants, optionVars) + // no need to load files since they are done at template level + m.options.TemplateCtx.Merge(allVars) // merge all variables into template context + return nil +} + +// ExecuteWithResults executes the template and returns results +func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + var finalProtoEvent *output.InternalWrappedEvent + // callback to process results from all protocols + multiProtoCallback := func(event *output.InternalWrappedEvent) { + if event != nil { + finalProtoEvent = event + } + // export dynamic values from operators (i.e internal:true) + if event.OperatorsResult != nil && len(event.OperatorsResult.DynamicValues) > 0 { + for k, v := range event.OperatorsResult.DynamicValues { + // TBD: iterate-all is only supported in `http` protocol + // we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context) + // currently if dynamic value array only contains one value we replace it with the value + if len(v) == 1 { + m.options.TemplateCtx.Set(k, v[0]) + } else { + // Note: if extracted value contains multiple values then they can be accessed by indexing + // ex: if values are dynamic = []string{"a","b","c"} then they are available as + // dynamic = "a" , dynamic1 = "b" , dynamic2 = "c" + // we intentionally omit first index for unknown situations (where no of extracted values are not known) + for i, val := range v { + if i == 0 { + m.options.TemplateCtx.Set(k, val) + } else { + m.options.TemplateCtx.Set(k+strconv.Itoa(i), val) + } + } + } + } + } + + // evaluate all variables after execution of each protocol + variableMap := m.options.Variables.Evaluate(m.options.TemplateCtx.GetAll()) + m.options.TemplateCtx.Merge(variableMap) // merge all variables into template context + } + + // template context: contains values extracted using `internal` extractor from previous protocols + // these values are extracted from each protocol in queue and are passed to next protocol in queue + // instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows) + // this makes it possible to use multi protocol templates in workflows + // Note: internal extractor values take precedence over dynamicValues from workflows (i.e other templates in workflow) + + // execute all protocols in the queue + for _, req := range m.requests { + values := m.options.TemplateCtx.GetAll() + err := req.ExecuteWithResults(input, output.InternalEvent(values), nil, multiProtoCallback) + // if error skip execution of next protocols + if err != nil { + return err + } + } + // Review: how to handle events of multiple protocols in a single template + // currently the outer callback is only executed once (for the last protocol in queue) + // due to workflow logic at https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/executer/executem.go#L150 + // this causes addition of duplicated / unncessary variables with prefix template_id_all_variables + callback(finalProtoEvent) + + return nil +} + +// Name of the template engine +func (m *MultiProtocol) Name() string { + return "multiproto" +} diff --git a/v2/pkg/protocols/multi/request_test.go b/v2/pkg/tmplexec/multiproto/multi_test.go similarity index 86% rename from v2/pkg/protocols/multi/request_test.go rename to v2/pkg/tmplexec/multiproto/multi_test.go index 660f69afe8..d09d02d739 100644 --- a/v2/pkg/protocols/multi/request_test.go +++ b/v2/pkg/tmplexec/multiproto/multi_test.go @@ -1,4 +1,4 @@ -package multi_test +package multiproto_test import ( "context" @@ -47,9 +47,9 @@ func TestMultiProtoWithDynamicExtractor(t *testing.T) { Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") - require.Equal(t, 2, len(Template.MultiProtoRequest.Queue)) + require.Equal(t, 2, len(Template.RequestsQueue)) - err = Template.MultiProtoRequest.Compile(&executerOpts) + err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) @@ -62,13 +62,11 @@ func TestMultiProtoWithProtoPrefix(t *testing.T) { Template, err := templates.Parse("testcases/multiprotowithprefix.yaml", nil, executerOpts) require.Nil(t, err, "could not parse template") - require.Equal(t, 3, len(Template.MultiProtoRequest.Queue)) + require.Equal(t, 3, len(Template.RequestsQueue)) - err = Template.MultiProtoRequest.Compile(&executerOpts) + err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") - require.True(t, len(Template.MultiProtoRequest.GetCompiledOperators()) > 0, "could not compile operators") - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) require.Nil(t, err, "could not execute template") require.True(t, gotresults) diff --git a/v2/pkg/protocols/multi/testcases/multiprotodynamic.yaml b/v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml similarity index 100% rename from v2/pkg/protocols/multi/testcases/multiprotodynamic.yaml rename to v2/pkg/tmplexec/multiproto/testcases/multiprotodynamic.yaml diff --git a/v2/pkg/protocols/multi/testcases/multiprotowithprefix.yaml b/v2/pkg/tmplexec/multiproto/testcases/multiprotowithprefix.yaml similarity index 100% rename from v2/pkg/protocols/multi/testcases/multiprotowithprefix.yaml rename to v2/pkg/tmplexec/multiproto/testcases/multiprotowithprefix.yaml From cd94876201d3b01b671c4845a14127a221715f1c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 22 Aug 2023 19:45:58 +0530 Subject: [PATCH 28/38] fix data race in templateCtx --- v2/pkg/protocols/protocols.go | 1 + v2/pkg/tmplexec/exec.go | 4 ++++ v2/pkg/tmplexec/flow/flow_executor.go | 9 ++++----- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index 8886de16a5..cfddf4dbe6 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -138,6 +138,7 @@ func (e *ExecutorOptions) AddTemplateVar(templateType templateTypes.ProtocolType // Copy returns a copy of the executeroptions structure func (e ExecutorOptions) Copy() ExecutorOptions { copy := e + copy.TemplateCtx = e.TemplateCtx.Clone() return copy } diff --git a/v2/pkg/tmplexec/exec.go b/v2/pkg/tmplexec/exec.go index 7371d163a7..14c7c6c340 100644 --- a/v2/pkg/tmplexec/exec.go +++ b/v2/pkg/tmplexec/exec.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "strings" + "sync" "sync/atomic" "github.com/projectdiscovery/gologger" @@ -23,6 +24,7 @@ type TemplateExecuter struct { options *protocols.ExecutorOptions engine TemplateEngine results *atomic.Bool + m sync.Mutex } // Both executer & Executor are correct spellings (its open to interpretation) @@ -131,7 +133,9 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { // and while executing we create new instance of flow executor everytime if e.options.Flow != "" { // compile flow + e.m.Lock() flowexec := flow.NewFlowExecutor(e.requests, e.options, results) + e.m.Unlock() if err := flowexec.Compile(); err != nil { return false, err } diff --git a/v2/pkg/tmplexec/flow/flow_executor.go b/v2/pkg/tmplexec/flow/flow_executor.go index c551856aea..bd6b2253e4 100644 --- a/v2/pkg/tmplexec/flow/flow_executor.go +++ b/v2/pkg/tmplexec/flow/flow_executor.go @@ -75,7 +75,7 @@ func NewFlowExecutor(requests []protocols.Request, options *protocols.ExecutorOp gologger.Error().Msgf("invalid request type %s", req.Type().String()) } } - return &FlowExecutor{ + f := &FlowExecutor{ allProtocols: allprotos, options: options, allErrs: mapsutil.SyncLockMap[string, error]{ @@ -84,7 +84,10 @@ func NewFlowExecutor(requests []protocols.Request, options *protocols.ExecutorOp }, protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{}, results: results, + jsVM: goja.New(), } + f.options.TemplateCtx = contextargs.New() + return f } // Init initializes the flow executor all dependencies @@ -94,10 +97,6 @@ func (f *FlowExecutor) Compile() error { if f.results == nil { f.results = new(atomic.Bool) } - // store all dynamic variables and other variables here - f.options.TemplateCtx = contextargs.New() - // create a new js vm/runtime - f.jsVM = goja.New() // load all variables and evaluate with existing data variableMap := f.options.Variables.Evaluate(f.options.TemplateCtx.GetAll()) From c57aee9591f6c6d784bfc26ed86d2c173b56107e Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 22 Aug 2023 23:57:26 +0530 Subject: [PATCH 29/38] templateCtx redesign --- v2/pkg/protocols/code/code.go | 6 +-- .../protocols/common/contextargs/metainput.go | 7 +++ v2/pkg/protocols/dns/request.go | 12 ++--- v2/pkg/protocols/file/request.go | 18 +++---- v2/pkg/protocols/headless/request.go | 6 +-- v2/pkg/protocols/http/build_request.go | 2 +- v2/pkg/protocols/http/request.go | 4 +- v2/pkg/protocols/network/request.go | 14 ++--- v2/pkg/protocols/offlinehttp/request.go | 4 +- v2/pkg/protocols/protocols.go | 53 +++++++++++++++---- v2/pkg/protocols/ssl/ssl.go | 8 +-- v2/pkg/protocols/websocket/websocket.go | 13 ++--- v2/pkg/protocols/whois/whois.go | 6 +-- v2/pkg/templates/compile.go | 5 +- v2/pkg/testutils/testutils.go | 3 +- v2/pkg/tmplexec/exec.go | 18 ++++--- v2/pkg/tmplexec/flow/flow_executor.go | 15 +++--- v2/pkg/tmplexec/flow/flow_executor_test.go | 25 +++++---- v2/pkg/tmplexec/flow/flow_internal.go | 20 +++---- v2/pkg/tmplexec/multiproto/README.md | 6 +-- v2/pkg/tmplexec/multiproto/multi.go | 28 +++++----- 21 files changed, 158 insertions(+), 115 deletions(-) diff --git a/v2/pkg/protocols/code/code.go b/v2/pkg/protocols/code/code.go index 14de35becb..f93af3fc93 100644 --- a/v2/pkg/protocols/code/code.go +++ b/v2/pkg/protocols/code/code.go @@ -118,7 +118,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa // inject all template context values as gozero env variables variables := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil) // add template context values - variables = generators.MergeMaps(variables, request.options.TemplateCtx.GetAll()) + variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) variablesMap := request.options.Variables.Evaluate(variables) @@ -157,10 +157,10 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa // expose response variables in proto_var format // this is no-op if the template is not a multi protocol template - request.options.AddTemplateVars(request.Type(), request.ID, data) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data) // add variables from template context before matching/extraction - data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) if request.options.Interactsh != nil { request.options.Interactsh.MakePlaceholders(interactshURLs, data) diff --git a/v2/pkg/protocols/common/contextargs/metainput.go b/v2/pkg/protocols/common/contextargs/metainput.go index b76603de16..3c30b82ff8 100644 --- a/v2/pkg/protocols/common/contextargs/metainput.go +++ b/v2/pkg/protocols/common/contextargs/metainput.go @@ -2,6 +2,7 @@ package contextargs import ( "bytes" + "crypto/md5" "fmt" "strings" @@ -67,3 +68,9 @@ func (metaInput *MetaInput) PrettyPrint() string { } return metaInput.Input } + +// GetScanHash returns a unique hash that represents a scan by hashing (metainput + templateId) +func (metaInput *MetaInput) GetScanHash(templateId string) string { + scanHash := md5.Sum([]byte(templateId + ":" + metaInput.Input + ":" + metaInput.CustomIP)) + return string(scanHash[:]) +} diff --git a/v2/pkg/protocols/dns/request.go b/v2/pkg/protocols/dns/request.go index 845ac9a816..c2e1fcdbed 100644 --- a/v2/pkg/protocols/dns/request.go +++ b/v2/pkg/protocols/dns/request.go @@ -53,7 +53,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // optionvars are vars passed from CLI or env variables optionVars := generators.BuildPayloadFromOptions(request.options.Options) // merge with metadata (eg. from workflow context) - vars = generators.MergeMaps(vars, metadata, optionVars, request.options.TemplateCtx.GetAll()) + vars = generators.MergeMaps(vars, metadata, optionVars, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(vars) vars = generators.MergeMaps(vars, variablesMap, request.options.Constants) @@ -66,18 +66,18 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, break } value = generators.MergeMaps(vars, value) - if err := request.execute(domain, metadata, previous, value, callback); err != nil { + if err := request.execute(input, domain, metadata, previous, value, callback); err != nil { return err } } } else { value := maps.Clone(vars) - return request.execute(domain, metadata, previous, value, callback) + return request.execute(input, domain, metadata, previous, value, callback) } return nil } -func (request *Request) execute(domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { +func (request *Request) execute(input *contextargs.Context, domain string, metadata, previous output.InternalEvent, vars map[string]interface{}, callback protocols.OutputEventCallback) error { if vardump.EnableVarDump { gologger.Debug().Msgf("Protocol request variables: \n%s\n", vardump.DumpVariables(vars)) @@ -151,7 +151,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna outputEvent := request.responseToDSLMap(compiledRequest, response, domain, question, traceData) // expose response variables in proto_var format // this is no-op if the template is not a multi protocol template - request.options.AddTemplateVars(request.Type(), request.ID, outputEvent) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) for k, v := range previous { outputEvent[k] = v } @@ -159,7 +159,7 @@ func (request *Request) execute(domain string, metadata, previous output.Interna outputEvent[k] = v } // add variables from template context before matching/extraction - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse) dumpResponse(event, request, request.options, response.String(), question) diff --git a/v2/pkg/protocols/file/request.go b/v2/pkg/protocols/file/request.go index dc19deccf6..324b5a1946 100644 --- a/v2/pkg/protocols/file/request.go +++ b/v2/pkg/protocols/file/request.go @@ -64,7 +64,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // every new file in the compressed multi-file archive counts 1 request.options.Progress.AddToTotal(1) archiveFileName := filepath.Join(filePath, file.Name()) - event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input.MetaInput.Input, file.Size(), previous) + event, fileMatches, err := request.processReader(file.ReadCloser, archiveFileName, input, file.Size(), previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -117,7 +117,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, _ = tmpFileOut.Sync() // rewind the file _, _ = tmpFileOut.Seek(0, 0) - event, fileMatches, err := request.processReader(tmpFileOut, filePath, input.MetaInput.Input, fileStat.Size(), previous) + event, fileMatches, err := request.processReader(tmpFileOut, filePath, input, fileStat.Size(), previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -137,7 +137,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, default: // normal file - increments the counter by 1 request.options.Progress.AddToTotal(1) - event, fileMatches, err := request.processFile(filePath, input.MetaInput.Input, previous) + event, fileMatches, err := request.processFile(filePath, input, previous) if err != nil { if errors.Is(err, errEmptyResult) { // no matches but one file elaborated @@ -166,7 +166,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, return nil } -func (request *Request) processFile(filePath, input string, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { +func (request *Request) processFile(filePath string, input *contextargs.Context, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { file, err := os.Open(filePath) if err != nil { return nil, nil, errors.Errorf("Could not open file path %s: %s\n", filePath, err) @@ -185,7 +185,7 @@ func (request *Request) processFile(filePath, input string, previousInternalEven return request.processReader(file, filePath, input, stat.Size(), previousInternalEvent) } -func (request *Request) processReader(reader io.Reader, filePath, input string, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { +func (request *Request) processReader(reader io.Reader, filePath string, input *contextargs.Context, totalBytes int64, previousInternalEvent output.InternalEvent) (*output.InternalWrappedEvent, []FileMatch, error) { fileReader := io.LimitReader(reader, request.maxSize) fileMatches, opResult := request.findMatchesWithReader(fileReader, input, filePath, totalBytes, previousInternalEvent) if opResult == nil && len(fileMatches) == 0 { @@ -193,10 +193,10 @@ func (request *Request) processReader(reader io.Reader, filePath, input string, } // build event structure to interface with internal logic - return request.buildEvent(input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil + return request.buildEvent(input.MetaInput.Input, filePath, fileMatches, opResult, previousInternalEvent), fileMatches, nil } -func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) { +func (request *Request) findMatchesWithReader(reader io.Reader, input *contextargs.Context, filePath string, totalBytes int64, previous output.InternalEvent) ([]FileMatch, *operators.Result) { var bytesCount, linesCount, wordsCount int isResponseDebug := request.options.Options.Debug || request.options.Options.DebugResponse totalBytesString := units.BytesSize(float64(totalBytes)) @@ -243,12 +243,12 @@ func (request *Request) findMatchesWithReader(reader io.Reader, input, filePath processedBytes := units.BytesSize(float64(currentBytes)) gologger.Verbose().Msgf("[%s] Processing file %s chunk %s/%s", request.options.TemplateID, filePath, processedBytes, totalBytesString) - dslMap := request.responseToDSLMap(lineContent, input, filePath) + dslMap := request.responseToDSLMap(lineContent, input.MetaInput.Input, filePath) for k, v := range previous { dslMap[k] = v } // add template context variables to DSL map - dslMap = generators.MergeMaps(dslMap, request.options.TemplateCtx.GetAll()) + dslMap = generators.MergeMaps(dslMap, request.options.GetTemplateCtx(input.MetaInput).GetAll()) discardEvent := eventcreator.CreateEvent(request, dslMap, isResponseDebug) newOpResult := discardEvent.OperatorsResult if newOpResult != nil { diff --git a/v2/pkg/protocols/headless/request.go b/v2/pkg/protocols/headless/request.go index eb0e18c004..24aec77abb 100644 --- a/v2/pkg/protocols/headless/request.go +++ b/v2/pkg/protocols/headless/request.go @@ -44,7 +44,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, vars := protocolutils.GenerateVariablesWithContextArgs(input, false) payloads := generators.BuildPayloadFromOptions(request.options.Options) // add templatecontext variables to varMap - values := generators.MergeMaps(vars, metadata, payloads, request.options.TemplateCtx.GetAll()) + values := generators.MergeMaps(vars, metadata, payloads, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(values) payloads = generators.MergeMaps(variablesMap, payloads, request.options.Constants) @@ -156,8 +156,8 @@ func (request *Request) executeRequestWithPayloads(input *contextargs.Context, p outputEvent := request.responseToDSLMap(responseBody, out["header"], out["status_code"], reqBuilder.String(), input.MetaInput.Input, input.MetaInput.Input, page.DumpHistory()) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), request.ID, outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) for k, v := range out { outputEvent[k] = v } diff --git a/v2/pkg/protocols/http/build_request.go b/v2/pkg/protocols/http/build_request.go index 06bccc252f..69661e3dbd 100644 --- a/v2/pkg/protocols/http/build_request.go +++ b/v2/pkg/protocols/http/build_request.go @@ -73,7 +73,7 @@ func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, // add template context values to dynamicValues (this takes care of self-contained and other types of requests) // Note: `iterate-all` and flow are mutually exclusive. flow uses templateCtx and iterate-all uses dynamicValues - dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.TemplateCtx.GetAll()) + dynamicValues = generators.MergeMaps(dynamicValues, r.request.options.GetTemplateCtx(input.MetaInput).GetAll()) if r.request.SelfContained { return r.makeSelfContainedRequest(ctx, reqData, payloads, dynamicValues) } diff --git a/v2/pkg/protocols/http/request.go b/v2/pkg/protocols/http/request.go index 88db9387e2..79a43a7d03 100644 --- a/v2/pkg/protocols/http/request.go +++ b/v2/pkg/protocols/http/request.go @@ -721,8 +721,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ outputEvent := request.responseToDSLMap(response.resp, input.MetaInput.Input, matchedURL, tostring.UnsafeToString(dumpedRequest), tostring.UnsafeToString(response.fullResponse), tostring.UnsafeToString(response.body), tostring.UnsafeToString(response.headers), duration, generatedRequest.meta) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), request.ID, outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index af7538c957..ad76e19c3d 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -56,7 +56,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } variables := protocolutils.GenerateVariables(address, false, nil) // add template ctx variables to varMap - variables = generators.MergeMaps(variables, request.options.TemplateCtx.GetAll()) + variables = generators.MergeMaps(variables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(variables) variables = generators.MergeMaps(variablesMap, variables, request.options.Constants) @@ -70,7 +70,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } visitedAddresses.Set(actualAddress, struct{}{}) - if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { + if err := request.executeAddress(variables, actualAddress, address, input, kv.tls, previous, callback); err != nil { outputEvent := request.responseToDSLMap("", "", "", address, "") callback(&output.InternalWrappedEvent{InternalEvent: outputEvent}) gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err) @@ -81,7 +81,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } // executeAddress executes the request for an address -func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeAddress(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, previous output.InternalEvent, callback protocols.OutputEventCallback) error { variables = generators.MergeMaps(variables, map[string]interface{}{"Hostname": address}) payloads := generators.BuildPayloadFromOptions(request.options.Options) @@ -114,7 +114,7 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA return nil } -func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address, input string, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(variables map[string]interface{}, actualAddress, address string, input *contextargs.Context, shouldUseTLS bool, payloads map[string]interface{}, previous output.InternalEvent, callback protocols.OutputEventCallback) error { var ( hostname string conn net.Conn @@ -279,10 +279,10 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac } response := responseBuilder.String() - outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input, actualAddress) + outputEvent := request.responseToDSLMap(reqBuilder.String(), string(final[:n]), response, input.MetaInput.Input, actualAddress) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), request.ID, outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) outputEvent["ip"] = request.dialer.GetDialedIP(hostname) if request.options.StopAtFirstMatch { outputEvent["stop-at-first-match"] = true diff --git a/v2/pkg/protocols/offlinehttp/request.go b/v2/pkg/protocols/offlinehttp/request.go index 58386c3a32..a942288aff 100644 --- a/v2/pkg/protocols/offlinehttp/request.go +++ b/v2/pkg/protocols/offlinehttp/request.go @@ -88,8 +88,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), utils.HeadersToString(resp.Header), 0, nil) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), request.GetID(), outputEvent) - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), outputEvent) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(input.MetaInput).GetAll()) outputEvent["ip"] = "" for k, v := range previous { outputEvent[k] = v diff --git a/v2/pkg/protocols/protocols.go b/v2/pkg/protocols/protocols.go index cfddf4dbe6..b894d7a164 100644 --- a/v2/pkg/protocols/protocols.go +++ b/v2/pkg/protocols/protocols.go @@ -1,7 +1,10 @@ package protocols import ( + "sync/atomic" + "github.com/projectdiscovery/ratelimit" + mapsutil "github.com/projectdiscovery/utils/maps" stringsutil "github.com/projectdiscovery/utils/strings" "github.com/logrusorgru/aurora" @@ -86,21 +89,47 @@ type ExecutorOptions struct { Colorizer aurora.Aurora WorkflowLoader model.WorkflowLoader ResumeCfg *types.ResumeCfg - // TemplateContext (contains all variables that are templatescoped i.e multi protocol) - // only used in case of multi protocol templates - TemplateCtx *contextargs.Context // ProtocolType is the type of the template ProtocolType templateTypes.ProtocolType // Flow is execution flow for the template (written in javascript) Flow string // IsMultiProtocol is true if template has more than one protocol IsMultiProtocol bool - // TemplateCtx retains state of template variables + // templateStore is a map which contains template context for each scan (i.e input * template-id pair) + templateCtxStore *mapsutil.SyncLockMap[string, *contextargs.Context] +} + +// CreateTemplateCtxStore creates template context store (which contains templateCtx for every scan) +func (e *ExecutorOptions) CreateTemplateCtxStore() { + e.templateCtxStore = &mapsutil.SyncLockMap[string, *contextargs.Context]{ + Map: make(map[string]*contextargs.Context), + ReadOnly: atomic.Bool{}, + } +} + +// RemoveTemplateCtx removes template context of given scan from store +func (e *ExecutorOptions) RemoveTemplateCtx(input *contextargs.MetaInput) { + scanId := input.GetScanHash(e.TemplateID) + if e.templateCtxStore != nil { + e.templateCtxStore.Delete(scanId) + } +} + +// GetTemplateCtx returns template context for given input +func (e *ExecutorOptions) GetTemplateCtx(input *contextargs.MetaInput) *contextargs.Context { + scanId := input.GetScanHash(e.TemplateID) + templateCtx, ok := e.templateCtxStore.Get(scanId) + if !ok { + // if template context does not exist create new and add it to store and return it + templateCtx = contextargs.New() + _ = e.templateCtxStore.Set(scanId, templateCtx) + } + return templateCtx } // AddTemplateVars adds vars to template context with given template type as prefix // this method is no-op if template is not multi protocol -func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolType, reqID string, vars map[string]interface{}) { +func (e *ExecutorOptions) AddTemplateVars(input *contextargs.MetaInput, reqType templateTypes.ProtocolType, reqID string, vars map[string]interface{}) { // if we wan't to disable adding response variables and other variables to template context // this is the statement that does it . template context is currently only enabled for // multiprotocol and flow templates @@ -108,37 +137,39 @@ func (e *ExecutorOptions) AddTemplateVars(templateType templateTypes.ProtocolTyp // no-op if not multi protocol template or flow template return } + templateCtx := e.GetTemplateCtx(input) for k, v := range vars { if !stringsutil.EqualFoldAny(k, "template-id", "template-info", "template-path") { if reqID != "" { k = reqID + "_" + k - } else if templateType < templateTypes.InvalidProtocol { - k = templateType.String() + "_" + k + } else if reqType < templateTypes.InvalidProtocol { + k = reqType.String() + "_" + k } - e.TemplateCtx.Set(k, v) + templateCtx.Set(k, v) } } } // AddTemplateVar adds given var to template context with given template type as prefix // this method is no-op if template is not multi protocol -func (e *ExecutorOptions) AddTemplateVar(templateType templateTypes.ProtocolType, reqID string, key string, value interface{}) { +func (e *ExecutorOptions) AddTemplateVar(input *contextargs.MetaInput, templateType templateTypes.ProtocolType, reqID string, key string, value interface{}) { if !e.IsMultiProtocol && e.Flow == "" { // no-op if not multi protocol template or flow template return } + templateCtx := e.GetTemplateCtx(input) if reqID != "" { key = reqID + "_" + key } else if templateType < templateTypes.InvalidProtocol { key = templateType.String() + "_" + key } - e.TemplateCtx.Set(key, value) + templateCtx.Set(key, value) } // Copy returns a copy of the executeroptions structure func (e ExecutorOptions) Copy() ExecutorOptions { copy := e - copy.TemplateCtx = e.TemplateCtx.Clone() + copy.CreateTemplateCtxStore() return copy } diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index f12f39eaf6..90b0563f95 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -187,7 +187,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa hostnameVariables := protocolutils.GenerateDNSVariables(hostname) // add template context variables to varMap - values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll()) + values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.GetTemplateCtx(input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(values) payloadValues = generators.MergeMaps(variablesMap, payloadValues, request.options.Constants) @@ -270,7 +270,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if tag == "" || f.IsZero() { continue } - request.options.AddTemplateVar(request.Type(), request.ID, tag, f.Value()) + request.options.AddTemplateVar(input.MetaInput, request.Type(), request.ID, tag, f.Value()) data[tag] = f.Value() } @@ -289,12 +289,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if tag == "" || f.IsZero() { continue } - request.options.AddTemplateVar(request.Type(), request.ID, tag, f.Value()) + request.options.AddTemplateVar(input.MetaInput, request.Type(), request.ID, tag, f.Value()) data[tag] = f.Value() } // add response fields ^ to template context and merge templatectx variables to output event - data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) event := eventcreator.CreateEvent(request, data, requestOptions.Options.Debug || requestOptions.Options.DebugResponse) if requestOptions.Options.Debug || requestOptions.Options.DebugResponse || requestOptions.Options.StoreResponse { msg := fmt.Sprintf("[%s] Dumped SSL response for %s", requestOptions.TemplateID, input.MetaInput.Input) diff --git a/v2/pkg/protocols/websocket/websocket.go b/v2/pkg/protocols/websocket/websocket.go index e65cad4498..4e8481a007 100644 --- a/v2/pkg/protocols/websocket/websocket.go +++ b/v2/pkg/protocols/websocket/websocket.go @@ -154,13 +154,13 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa if !ok { break } - if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { return err } } } else { value := make(map[string]interface{}) - if err := request.executeRequestWithPayloads(input.MetaInput.Input, hostname, value, previous, callback); err != nil { + if err := request.executeRequestWithPayloads(input, hostname, value, previous, callback); err != nil { return err } } @@ -168,8 +168,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. -func (request *Request) executeRequestWithPayloads(input, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeRequestWithPayloads(target *contextargs.Context, hostname string, dynamicValues, previous output.InternalEvent, callback protocols.OutputEventCallback) error { header := http.Header{} + input := target.MetaInput.Input parsed, err := urlutil.Parse(input) if err != nil { @@ -178,7 +179,7 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam defaultVars := protocolutils.GenerateVariables(parsed, false, nil) optionVars := generators.BuildPayloadFromOptions(request.options.Options) // add templatecontext variables to varMap - variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll())) + variables := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(target.MetaInput).GetAll())) payloadValues := generators.MergeMaps(variables, defaultVars, optionVars, dynamicValues, request.options.Constants) requestOptions := request.options @@ -267,8 +268,8 @@ func (request *Request) executeRequestWithPayloads(input, hostname string, dynam data["ip"] = request.dialer.GetDialedIP(hostname) // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), request.ID, data) - data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(target.MetaInput, request.Type(), request.ID, data) + data = generators.MergeMaps(data, request.options.GetTemplateCtx(target.MetaInput).GetAll()) for k, v := range previous { data[k] = v diff --git a/v2/pkg/protocols/whois/whois.go b/v2/pkg/protocols/whois/whois.go index 942eb3ba53..ac9ae7828b 100644 --- a/v2/pkg/protocols/whois/whois.go +++ b/v2/pkg/protocols/whois/whois.go @@ -94,7 +94,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa defaultVars := protocolutils.GenerateVariables(input.MetaInput.Input, false, nil) optionVars := generators.BuildPayloadFromOptions(request.options.Options) // add templatectx variables to varMap - vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.TemplateCtx.GetAll())) + vars := request.options.Variables.Evaluate(generators.MergeMaps(defaultVars, optionVars, dynamicValues, request.options.GetTemplateCtx(input.MetaInput).GetAll())) variables := generators.MergeMaps(vars, defaultVars, optionVars, dynamicValues, request.options.Constants) @@ -137,8 +137,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa data["response"] = jsonDataString // add response fields to template context and merge templatectx variables to output event - request.options.AddTemplateVars(request.Type(), request.ID, data) - data = generators.MergeMaps(data, request.options.TemplateCtx.GetAll()) + request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, data) + data = generators.MergeMaps(data, request.options.GetTemplateCtx(input.MetaInput).GetAll()) event := eventcreator.CreateEvent(request, data, request.options.Options.Debug || request.options.Options.DebugResponse) if request.options.Options.Debug || request.options.Options.DebugResponse { diff --git a/v2/pkg/templates/compile.go b/v2/pkg/templates/compile.go index a67a93c103..0b2d643ca3 100644 --- a/v2/pkg/templates/compile.go +++ b/v2/pkg/templates/compile.go @@ -12,7 +12,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/operators" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/offlinehttp" "github.com/projectdiscovery/nuclei/v2/pkg/templates/cache" "github.com/projectdiscovery/nuclei/v2/pkg/templates/signer" @@ -63,7 +62,7 @@ func Parse(filePath string, preprocessor Preprocessor, options protocols.Executo } defer reader.Close() options.TemplatePath = filePath - template, err := ParseTemplateFromReader(reader, preprocessor, options) + template, err := ParseTemplateFromReader(reader, preprocessor, options.Copy()) if err != nil { return nil, err } @@ -275,7 +274,7 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option } // create empty context args for template scope - options.TemplateCtx = contextargs.New() + options.CreateTemplateCtxStore() options.ProtocolType = template.Type() options.Constants = template.Constants diff --git a/v2/pkg/testutils/testutils.go b/v2/pkg/testutils/testutils.go index 616e0c5ebf..893c9af08c 100644 --- a/v2/pkg/testutils/testutils.go +++ b/v2/pkg/testutils/testutils.go @@ -16,7 +16,6 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/output" "github.com/projectdiscovery/nuclei/v2/pkg/progress" "github.com/projectdiscovery/nuclei/v2/pkg/protocols" - "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/protocolinit" "github.com/projectdiscovery/nuclei/v2/pkg/types" ) @@ -91,8 +90,8 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco Browser: nil, Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - TemplateCtx: contextargs.New(), } + executerOpts.CreateTemplateCtxStore() return executerOpts } diff --git a/v2/pkg/tmplexec/exec.go b/v2/pkg/tmplexec/exec.go index 14c7c6c340..aa2baf0bd5 100644 --- a/v2/pkg/tmplexec/exec.go +++ b/v2/pkg/tmplexec/exec.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "strings" - "sync" "sync/atomic" "github.com/projectdiscovery/gologger" @@ -24,7 +23,6 @@ type TemplateExecuter struct { options *protocols.ExecutorOptions engine TemplateEngine results *atomic.Bool - m sync.Mutex } // Both executer & Executor are correct spellings (its open to interpretation) @@ -45,7 +43,10 @@ func NewTemplateExecuter(requests []protocols.Request, options *protocols.Execut e := &TemplateExecuter{requests: requests, options: options, results: &atomic.Bool{}} if options.Flow != "" { - e.engine = flow.NewFlowExecutor(requests, options, e.results) + // we use a dummy input here because goal of flow executor at this point is to just check + // syntax and other things are correct before proceeding to actual execution + // during execution new instance of flow will be created as it is tightly coupled with lot of executor options + e.engine = flow.NewFlowExecutor(requests, contextargs.NewWithInput("dummy"), options, e.results) } else { // Review: // multiproto engine is only used if there is more than one protocol in template @@ -94,6 +95,11 @@ func (e *TemplateExecuter) Requests() int { // Execute executes the protocol group and returns true or false if results were found. func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { results := &atomic.Bool{} + defer func() { + // it is essential to remove template context of `Scan i.e template x input pair` + // since it is of no use after scan is completed (regardless of success or failure) + e.options.RemoveTemplateCtx(input.MetaInput) + }() var lastMatcherEvent *output.InternalWrappedEvent writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { @@ -132,14 +138,10 @@ func (e *TemplateExecuter) Execute(input *contextargs.Context) (bool, error) { // so in compile step earlier we compile it to validate javascript syntax and other things // and while executing we create new instance of flow executor everytime if e.options.Flow != "" { - // compile flow - e.m.Lock() - flowexec := flow.NewFlowExecutor(e.requests, e.options, results) - e.m.Unlock() + flowexec := flow.NewFlowExecutor(e.requests, input, e.options, results) if err := flowexec.Compile(); err != nil { return false, err } - // execute flow err = flowexec.ExecuteWithResults(input, cliExecutorCallback) } else { err = e.engine.ExecuteWithResults(input, cliExecutorCallback) diff --git a/v2/pkg/tmplexec/flow/flow_executor.go b/v2/pkg/tmplexec/flow/flow_executor.go index bd6b2253e4..06abdc3243 100644 --- a/v2/pkg/tmplexec/flow/flow_executor.go +++ b/v2/pkg/tmplexec/flow/flow_executor.go @@ -49,7 +49,7 @@ type FlowExecutor struct { } // NewFlowExecutor creates a new flow executor from a list of requests -func NewFlowExecutor(requests []protocols.Request, options *protocols.ExecutorOptions, results *atomic.Bool) *FlowExecutor { +func NewFlowExecutor(requests []protocols.Request, input *contextargs.Context, options *protocols.ExecutorOptions, results *atomic.Bool) *FlowExecutor { allprotos := make(map[string][]protocols.Request) for _, req := range requests { switch req.Type() { @@ -85,21 +85,18 @@ func NewFlowExecutor(requests []protocols.Request, options *protocols.ExecutorOp protoFunctions: map[string]func(call goja.FunctionCall) goja.Value{}, results: results, jsVM: goja.New(), + input: input, } - f.options.TemplateCtx = contextargs.New() return f } -// Init initializes the flow executor all dependencies -// this compiles and prepares for execution of a flow -// since it has dependencies on variables and etc it can't be done moved to templates package +// Compile compiles js program and registers all functions func (f *FlowExecutor) Compile() error { if f.results == nil { f.results = new(atomic.Bool) } - // load all variables and evaluate with existing data - variableMap := f.options.Variables.Evaluate(f.options.TemplateCtx.GetAll()) + variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) // cli options optionVars := generators.BuildPayloadFromOptions(f.options.Options) // constants @@ -116,7 +113,7 @@ func (f *FlowExecutor) Compile() error { } } } - f.options.TemplateCtx.Merge(allVars) // merge all variables into template context + f.options.GetTemplateCtx(f.input.MetaInput).Merge(allVars) // merge all variables into template context // ---- define callback functions/objects---- f.protoFunctions = map[string]func(call goja.FunctionCall) goja.Value{} @@ -182,7 +179,7 @@ func (f *FlowExecutor) ExecuteWithResults(input *contextargs.Context, callback p // add all input args to template context if f.input != nil && f.input.HasArgs() { f.input.ForEach(func(key string, value interface{}) { - f.options.TemplateCtx.Set(key, value) + f.options.GetTemplateCtx(f.input.MetaInput).Set(key, value) }) } if f.callback == nil { diff --git a/v2/pkg/tmplexec/flow/flow_executor_test.go b/v2/pkg/tmplexec/flow/flow_executor_test.go index b946cb781c..dcb3889d41 100644 --- a/v2/pkg/tmplexec/flow/flow_executor_test.go +++ b/v2/pkg/tmplexec/flow/flow_executor_test.go @@ -53,11 +53,12 @@ func TestFlowTemplateWithIndex(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("hackerone.com")) + input := contextargs.NewWithInput("hackerone.com") + gotresults, err := Template.Executer.Execute(input) require.Nil(t, err, "could not execute template") require.True(t, gotresults) // apart from parse->compile->execution this testcase checks if dynamic extracted variables are available - value, ok := Template.Options.TemplateCtx.Get("nameservers") + value, ok := Template.Options.GetTemplateCtx(input.MetaInput).Get("nameservers") require.True(t, ok) if value != nil { require.True(t, len(value.([]string)) > 0) @@ -76,11 +77,12 @@ func TestFlowTemplateWithID(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("hackerone.com")) + target := contextargs.NewWithInput("hackerone.com") + gotresults, err := Template.Executer.Execute(target) require.Nil(t, err, "could not execute template") require.True(t, gotresults) - value, ok := Template.Options.TemplateCtx.Get("nameservers") + value, ok := Template.Options.GetTemplateCtx(target.MetaInput).Get("nameservers") require.True(t, ok) if value != nil { require.True(t, len(value.([]string)) > 0) @@ -102,7 +104,8 @@ func TestFlowWithProtoPrefix(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("hackerone.com")) + input := contextargs.NewWithInput("hackerone.com") + gotresults, err := Template.Executer.Execute(input) require.Nil(t, err, "could not execute template") require.True(t, gotresults) @@ -111,7 +114,7 @@ func TestFlowWithProtoPrefix(t *testing.T) { "probe-ns_host", "probe-ns_matched", "probe-ns_answer", "probe-ns_raw"} for _, v := range protoVars { - value, ok := Template.Options.TemplateCtx.Get(v) + value, ok := Template.Options.GetTemplateCtx(input.MetaInput).Get(v) require.Truef(t, ok, "could not find variable %s", v) if value != nil { require.Truef(t, len(value.(string)) > 0, "variable %s is empty", v) @@ -132,12 +135,13 @@ func TestFlowWithConditionNegative(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") + input := contextargs.NewWithInput("scanme.sh") // expect no results and verify thant dns request is executed and http is not - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("scanme.sh")) + gotresults, err := Template.Executer.Execute(input) require.Nil(t, err, "could not execute template") require.False(t, gotresults) - m := Template.Options.TemplateCtx.GetAll() + m := Template.Options.GetTemplateCtx(input.MetaInput).GetAll() require.Equal(t, m["http_status"], nil) // since http() was not execute this variable should not exist require.NotEqual(t, m["dns_raw"], "") // since dns() was execute this variable should exist @@ -156,12 +160,13 @@ func TestFlowWithConditionPositive(t *testing.T) { err = Template.Executer.Compile() require.Nil(t, err, "could not compile template") + input := contextargs.NewWithInput("blog.projectdiscovery.io") // positive match . expect results also verify that both dns() and http() were executed - gotresults, err := Template.Executer.Execute(contextargs.NewWithInput("blog.projectdiscovery.io")) + gotresults, err := Template.Executer.Execute(input) require.Nil(t, err, "could not execute template") require.True(t, gotresults) - m := Template.Options.TemplateCtx.GetAll() + m := Template.Options.GetTemplateCtx(input.MetaInput).GetAll() require.NotEqual(t, m["http_status"], "") // since http() was not execute this variable should not exist require.NotEqual(t, m["dns_raw"], "") // since dns() was execute this variable should exist diff --git a/v2/pkg/tmplexec/flow/flow_internal.go b/v2/pkg/tmplexec/flow/flow_internal.go index ad594adb94..6e27fa1619 100644 --- a/v2/pkg/tmplexec/flow/flow_internal.go +++ b/v2/pkg/tmplexec/flow/flow_internal.go @@ -21,11 +21,11 @@ import ( func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Request], opts *ProtoOptions) bool { defer func() { // evaluate all variables after execution of each protocol - variableMap := f.options.Variables.Evaluate(f.options.TemplateCtx.GetAll()) - f.options.TemplateCtx.Merge(variableMap) // merge all variables into template context + variableMap := f.options.Variables.Evaluate(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) + f.options.GetTemplateCtx(f.input.MetaInput).Merge(variableMap) // merge all variables into template context // to avoid polling update template variables everytime we execute a protocol - var m map[string]interface{} = f.options.TemplateCtx.GetAll() + var m map[string]interface{} = f.options.GetTemplateCtx(f.input.MetaInput).GetAll() _ = f.jsVM.Set("template", m) }() matcherStatus := &atomic.Bool{} // due to interactsh matcher polling logic this needs to be atomic bool @@ -34,7 +34,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req // execution logic for http()/dns() etc for index := range f.allProtocols[opts.protoName] { req := f.allProtocols[opts.protoName][index] - err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) { if result != nil { f.results.CompareAndSwap(false, true) if !opts.Hide { @@ -52,7 +52,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req } if len(result.OperatorsResult.DynamicValues) > 0 { for k, v := range result.OperatorsResult.DynamicValues { - f.options.TemplateCtx.Set(k, v) + f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v) } } } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { @@ -90,7 +90,7 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req } return matcherStatus.Load() } - err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.TemplateCtx.GetAll()), nil, func(result *output.InternalWrappedEvent) { + err := req.ExecuteWithResults(f.input, output.InternalEvent(f.options.GetTemplateCtx(f.input.MetaInput).GetAll()), nil, func(result *output.InternalWrappedEvent) { if result != nil { f.results.CompareAndSwap(false, true) if !opts.Hide { @@ -102,9 +102,9 @@ func (f *FlowExecutor) requestExecutor(reqMap mapsutil.Map[string, protocols.Req matcherStatus.CompareAndSwap(false, result.OperatorsResult.Matched) if len(result.OperatorsResult.DynamicValues) > 0 { for k, v := range result.OperatorsResult.DynamicValues { - f.options.TemplateCtx.Set(k, v) + f.options.GetTemplateCtx(f.input.MetaInput).Set(k, v) } - _ = f.jsVM.Set("template", f.options.TemplateCtx.GetAll()) + _ = f.jsVM.Set("template", f.options.GetTemplateCtx(f.input.MetaInput).GetAll()) } } } @@ -150,7 +150,7 @@ func (f *FlowExecutor) registerBuiltInFunctions() error { if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value { varName := call.Argument(0).Export() varValue := call.Argument(1).Export() - f.options.TemplateCtx.Set(varName.(string), varValue) + f.options.GetTemplateCtx(f.input.MetaInput).Set(varName.(string), varValue) return goja.Null() }); err != nil { return err @@ -225,7 +225,7 @@ func (f *FlowExecutor) registerBuiltInFunctions() error { return err } - var m = f.options.TemplateCtx.GetAll() + var m = f.options.GetTemplateCtx(f.input.MetaInput).GetAll() if m == nil { m = map[string]interface{}{} } diff --git a/v2/pkg/tmplexec/multiproto/README.md b/v2/pkg/tmplexec/multiproto/README.md index 09fc549fc1..7f874a8aed 100644 --- a/v2/pkg/tmplexec/multiproto/README.md +++ b/v2/pkg/tmplexec/multiproto/README.md @@ -16,10 +16,10 @@ which takes care of sync and other issues if any. all response fields are prefix while logic/implementation of multi protocol execution is abstracted. it requires 3 statements to be added in newly implemented protocol to make response fields of that protocol available to global context -- Add `request.options.TemplateCtx.GetAll()` to variablesMap in `ExecuteWithResults` Method just above `request.options.Variables.Evaluate` +- Add `request.options.GetTemplateCtx(f.input.MetaInput).GetAll()` to variablesMap in `ExecuteWithResults` Method just above `request.options.Variables.Evaluate` ```go // example - values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.TemplateCtx.GetAll()) + values := generators.MergeMaps(payloadValues, hostnameVariables, request.options.GetTemplateCtx(f.input.MetaInput).GetAll()) variablesMap := request.options.Variables.Evaluate(values) ``` @@ -34,7 +34,7 @@ to make response fields of that protocol available to global context - Append all available template context values to outputEvent ```go // add variables from template context before matching/extraction - outputEvent = generators.MergeMaps(outputEvent, request.options.TemplateCtx.GetAll()) + outputEvent = generators.MergeMaps(outputEvent, request.options.GetTemplateCtx(f.input.MetaInput).GetAll()) ``` adding these 3 statements takes care of all logic related to multi protocol execution diff --git a/v2/pkg/tmplexec/multiproto/multi.go b/v2/pkg/tmplexec/multiproto/multi.go index 68d2eb9bc4..9a4f37b68b 100644 --- a/v2/pkg/tmplexec/multiproto/multi.go +++ b/v2/pkg/tmplexec/multiproto/multi.go @@ -13,9 +13,10 @@ import ( // Mutliprotocol is a template executer engine that executes multiple protocols // with logic in between type MultiProtocol struct { - requests []protocols.Request - options *protocols.ExecutorOptions - results *atomic.Bool + requests []protocols.Request + options *protocols.ExecutorOptions + results *atomic.Bool + readOnlyArgs map[string]interface{} // readOnlyArgs are readonly args that are available after compilation } // NewMultiProtocol creates a new multiprotocol template engine from a list of requests @@ -28,22 +29,23 @@ func NewMultiProtocol(requests []protocols.Request, options *protocols.ExecutorO // Compile engine specific compilation func (m *MultiProtocol) Compile() error { - // protocol/ request is already handled by template executer - m.options.TemplateCtx = contextargs.New() // load all variables and evaluate with existing data - variableMap := m.options.Variables.Evaluate(m.options.TemplateCtx.GetAll()) + variableMap := m.options.Variables.GetAll() // cli options optionVars := generators.BuildPayloadFromOptions(m.options.Options) // constants constants := m.options.Constants allVars := generators.MergeMaps(variableMap, constants, optionVars) + allVars = m.options.Variables.Evaluate(allVars) + m.readOnlyArgs = allVars // no need to load files since they are done at template level - m.options.TemplateCtx.Merge(allVars) // merge all variables into template context return nil } // ExecuteWithResults executes the template and returns results func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback protocols.OutputEventCallback) error { + // put all readonly args into template context + m.options.GetTemplateCtx(input.MetaInput).Merge(m.readOnlyArgs) var finalProtoEvent *output.InternalWrappedEvent // callback to process results from all protocols multiProtoCallback := func(event *output.InternalWrappedEvent) { @@ -57,7 +59,7 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback // we either need to add support for iterate-all in other protocols or implement a different logic (specific to template context) // currently if dynamic value array only contains one value we replace it with the value if len(v) == 1 { - m.options.TemplateCtx.Set(k, v[0]) + m.options.GetTemplateCtx(input.MetaInput).Set(k, v[0]) } else { // Note: if extracted value contains multiple values then they can be accessed by indexing // ex: if values are dynamic = []string{"a","b","c"} then they are available as @@ -65,9 +67,9 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback // we intentionally omit first index for unknown situations (where no of extracted values are not known) for i, val := range v { if i == 0 { - m.options.TemplateCtx.Set(k, val) + m.options.GetTemplateCtx(input.MetaInput).Set(k, val) } else { - m.options.TemplateCtx.Set(k+strconv.Itoa(i), val) + m.options.GetTemplateCtx(input.MetaInput).Set(k+strconv.Itoa(i), val) } } } @@ -75,8 +77,8 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback } // evaluate all variables after execution of each protocol - variableMap := m.options.Variables.Evaluate(m.options.TemplateCtx.GetAll()) - m.options.TemplateCtx.Merge(variableMap) // merge all variables into template context + variableMap := m.options.Variables.Evaluate(m.options.GetTemplateCtx(input.MetaInput).GetAll()) + m.options.GetTemplateCtx(input.MetaInput).Merge(variableMap) // merge all variables into template context } // template context: contains values extracted using `internal` extractor from previous protocols @@ -87,7 +89,7 @@ func (m *MultiProtocol) ExecuteWithResults(input *contextargs.Context, callback // execute all protocols in the queue for _, req := range m.requests { - values := m.options.TemplateCtx.GetAll() + values := m.options.GetTemplateCtx(input.MetaInput).GetAll() err := req.ExecuteWithResults(input, output.InternalEvent(values), nil, multiProtoCallback) // if error skip execution of next protocols if err != nil { From 7de01ead78a19008fc659e339bfa92ddff69f06d Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 23 Aug 2023 00:05:40 +0530 Subject: [PATCH 30/38] fix failing unit test --- v2/pkg/tmplexec/flow/flow_executor_test.go | 34 ---------------------- 1 file changed, 34 deletions(-) diff --git a/v2/pkg/tmplexec/flow/flow_executor_test.go b/v2/pkg/tmplexec/flow/flow_executor_test.go index dcb3889d41..7c7ec983c5 100644 --- a/v2/pkg/tmplexec/flow/flow_executor_test.go +++ b/v2/pkg/tmplexec/flow/flow_executor_test.go @@ -57,12 +57,6 @@ func TestFlowTemplateWithIndex(t *testing.T) { gotresults, err := Template.Executer.Execute(input) require.Nil(t, err, "could not execute template") require.True(t, gotresults) - // apart from parse->compile->execution this testcase checks if dynamic extracted variables are available - value, ok := Template.Options.GetTemplateCtx(input.MetaInput).Get("nameservers") - require.True(t, ok) - if value != nil { - require.True(t, len(value.([]string)) > 0) - } } func TestFlowTemplateWithID(t *testing.T) { @@ -81,12 +75,6 @@ func TestFlowTemplateWithID(t *testing.T) { gotresults, err := Template.Executer.Execute(target) require.Nil(t, err, "could not execute template") require.True(t, gotresults) - - value, ok := Template.Options.GetTemplateCtx(target.MetaInput).Get("nameservers") - require.True(t, ok) - if value != nil { - require.True(t, len(value.([]string)) > 0) - } } func TestFlowWithProtoPrefix(t *testing.T) { @@ -108,18 +96,6 @@ func TestFlowWithProtoPrefix(t *testing.T) { gotresults, err := Template.Executer.Execute(input) require.Nil(t, err, "could not execute template") require.True(t, gotresults) - - // while there are lot of variables lets just look for only these - protoVars := []string{"dns_0_host", "dns_0_matched", "dns_0_answer", "dns_0_raw", - "probe-ns_host", "probe-ns_matched", "probe-ns_answer", "probe-ns_raw"} - - for _, v := range protoVars { - value, ok := Template.Options.GetTemplateCtx(input.MetaInput).Get(v) - require.Truef(t, ok, "could not find variable %s", v) - if value != nil { - require.Truef(t, len(value.(string)) > 0, "variable %s is empty", v) - } - } } func TestFlowWithConditionNegative(t *testing.T) { @@ -140,11 +116,6 @@ func TestFlowWithConditionNegative(t *testing.T) { gotresults, err := Template.Executer.Execute(input) require.Nil(t, err, "could not execute template") require.False(t, gotresults) - - m := Template.Options.GetTemplateCtx(input.MetaInput).GetAll() - - require.Equal(t, m["http_status"], nil) // since http() was not execute this variable should not exist - require.NotEqual(t, m["dns_raw"], "") // since dns() was execute this variable should exist } func TestFlowWithConditionPositive(t *testing.T) { @@ -165,11 +136,6 @@ func TestFlowWithConditionPositive(t *testing.T) { gotresults, err := Template.Executer.Execute(input) require.Nil(t, err, "could not execute template") require.True(t, gotresults) - - m := Template.Options.GetTemplateCtx(input.MetaInput).GetAll() - - require.NotEqual(t, m["http_status"], "") // since http() was not execute this variable should not exist - require.NotEqual(t, m["dns_raw"], "") // since dns() was execute this variable should exist } func TestFlowWithNoMatchers(t *testing.T) { From 5dfff5b0299427c6408db772dd3da749afeb4aa2 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 23 Aug 2023 00:32:38 +0530 Subject: [PATCH 31/38] add multiprotocol support to deprecated syntax --- v2/pkg/templates/templates.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index baf147d97b..eb613b9eea 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -334,6 +334,11 @@ func (template *Template) addRequestsToQueue(keys ...string) { template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsWHOIS)...) case types.CodeProtocol.String(): template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsCode)...) + // for deprecated protocols + case "requests": + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsHTTP)...) + case "network": + template.RequestsQueue = append(template.RequestsQueue, template.convertRequestToProtocolsRequest(template.RequestsNetwork)...) } } } From 286ede496043e81c0707338f73ba5f4272709ea3 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 23 Aug 2023 01:24:54 +0530 Subject: [PATCH 32/38] fix race condition in utils & tlsx --- .../protocols/code/py-env-var.yaml | 2 +- integration_tests/protocols/code/py-file.yaml | 2 +- .../protocols/code/py-interactsh.yaml | 2 +- .../protocols/code/py-snippet.yaml | 2 +- v2/go.mod | 18 ++++---- v2/go.sum | 43 ++++++++----------- 6 files changed, 32 insertions(+), 37 deletions(-) diff --git a/integration_tests/protocols/code/py-env-var.yaml b/integration_tests/protocols/code/py-env-var.yaml index 7ee2654bcc..b05ccf0923 100644 --- a/integration_tests/protocols/code/py-env-var.yaml +++ b/integration_tests/protocols/code/py-env-var.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input baz" -# digest: 4b0a00483046022100bb80bf0ba533906fe86a0436b427d46e1ebe9627054d7e334b017c7532909b8a0221009346e92aea683e1fea54195081abf084ca5c2ade85566d610aea4e2368d07526 \ No newline at end of file +# digest: 4a0a00473045022100e17c7a809fd64419baf401b5331edab3a68a4c182f7777614beb1862eb6ea8b7022011b95fc0e22d7f82e08e01b56ce87afdbe03027c238ba290a058d695226173ae \ No newline at end of file diff --git a/integration_tests/protocols/code/py-file.yaml b/integration_tests/protocols/code/py-file.yaml index 99294d74f8..ee76c50c32 100644 --- a/integration_tests/protocols/code/py-file.yaml +++ b/integration_tests/protocols/code/py-file.yaml @@ -18,4 +18,4 @@ code: - type: word words: - "hello from input" -# digest: 4a0a0047304502207ef2ec2a351f2d0bc77bb4792abd297cdb6e8735a45669a3d0848a3d833a47b7022100d3f9e0d5017c0c465fedc999d14ae61ba64c804a04d281bac540faa321f4ae74 \ No newline at end of file +# digest: 490a004630440220241d7faae14ab5760dbe7acf621a3923d0650148bc14a52a9be06ba76e8e0abf02201885fcc432d354d3c99ea97b964838719451bc97f148229f187f61eee7525eb6 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-interactsh.yaml b/integration_tests/protocols/code/py-interactsh.yaml index 2b99fabdbb..664c2d7e8f 100644 --- a/integration_tests/protocols/code/py-interactsh.yaml +++ b/integration_tests/protocols/code/py-interactsh.yaml @@ -26,4 +26,4 @@ code: part: interactsh_protocol words: - "http" -# digest: 4a0a0047304502207de0206dcd3be754fa65f2c2f93106faa5c4b5d9e04b521b12c5baabe3e5246c022100d2eccbe8e135fb55b93555028b8d92928a50afb1ddb20137d891fe53ca7048e1 \ No newline at end of file +# digest: 490a004630440220427cb7100f0b7d95224f490a6f4f626186782cb26c69f2551d6aefcdbc7c17d20220206161ad3a98afe8fcef9dd06d9a6dd5f34c5f7e3cd3ab7f81328f033dcd2b48 \ No newline at end of file diff --git a/integration_tests/protocols/code/py-snippet.yaml b/integration_tests/protocols/code/py-snippet.yaml index 59ac2fe0b5..6b77ca5227 100644 --- a/integration_tests/protocols/code/py-snippet.yaml +++ b/integration_tests/protocols/code/py-snippet.yaml @@ -20,4 +20,4 @@ code: - type: word words: - "hello from input" -# digest: 490a0046304402202557bb272aca2b21cfafa5cd82b519dd23d4490b2f34ed602529508fb1aeeead02207c613b01ce297d8b60295a3ba2318787564518ab76fef2d013d96e26f08829c6 \ No newline at end of file +# digest: 4a0a00473045022056092462597e85139626656d37df123094cb3861bdf583450c38814bac8df9cb022100e83a8c552f8f8a098f6b7ec8a32c6b448b995e000884beadb50cb0f2720117de \ No newline at end of file diff --git a/v2/go.mod b/v2/go.mod index 34fc7753fe..543f63117f 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -26,12 +26,12 @@ require ( github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.18 github.com/projectdiscovery/retryabledns v1.0.32 - github.com/projectdiscovery/retryablehttp-go v1.0.20 + github.com/projectdiscovery/retryablehttp-go v1.0.22 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 - github.com/shirou/gopsutil/v3 v3.23.6 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.5.1 github.com/syndtr/goleveldb v1.0.0 @@ -39,8 +39,8 @@ require ( github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db github.com/xanzy/go-gitlab v0.84.0 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.12.0 - golang.org/x/oauth2 v0.10.0 + golang.org/x/net v0.14.0 + golang.org/x/oauth2 v0.11.0 golang.org/x/text v0.12.0 gopkg.in/yaml.v2 v2.4.0 moul.io/http2curl v1.0.0 @@ -69,7 +69,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/projectdiscovery/dsl v0.0.16 github.com/projectdiscovery/fasttemplate v0.0.2 - github.com/projectdiscovery/goflags v0.1.12 + github.com/projectdiscovery/goflags v0.1.17 github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 github.com/projectdiscovery/httpx v1.3.4 @@ -77,9 +77,9 @@ require ( github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.1 + github.com/projectdiscovery/tlsx v1.1.4-0.20230822194609-b80759b2fa87 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 + github.com/projectdiscovery/utils v0.0.51-0.20230822193513-49253312211f github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -220,10 +220,10 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/zap v1.24.0 // indirect goftp.io/server/v2 v2.0.0 // indirect - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 golang.org/x/mod v0.12.0 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/sys v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.11.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/v2/go.sum b/v2/go.sum index 4eb1486264..9370c2180e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -150,16 +150,12 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= -github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20230707174833-636fdf960de1 h1:sC/DYk3eEi5cKkpJX1vl+CpAM138dmuW7rutje9Eo4E= -github.com/dop251/goja v0.0.0-20230707174833-636fdf960de1/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja v0.0.0-20230812105242-81d76064690d h1:9aaGwVf4q+kknu+mROAXUApJ1DoOwhE8dGj/XLBYzWg= github.com/dop251/goja v0.0.0-20230812105242-81d76064690d/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= @@ -250,7 +246,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= @@ -426,8 +421,8 @@ github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOi github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= -github.com/projectdiscovery/goflags v0.1.12 h1:NucjSqw7reczmon2vQq9KyOrvOmlnznECeifHI2gOW0= -github.com/projectdiscovery/goflags v0.1.12/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= +github.com/projectdiscovery/goflags v0.1.17 h1:Fr+28b7y5K3/aiSfN97X7jTtk0XcGgMKaT8I2X+7ZYI= +github.com/projectdiscovery/goflags v0.1.17/go.mod h1:n0TWe/lpb7AZCNagDRX9VPKB1gY+/o+EfD2zd8lJJJk= github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 h1:M74WAoZ99q/LJPHC8aIWIt8+FLh699KqLm2CUSHoytA= @@ -450,17 +445,19 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.32 h1:Ekr+1j1jwQ2qINW7T02uMcXFc3QeduN3vOligpfQgeo= github.com/projectdiscovery/retryabledns v1.0.32/go.mod h1:t8aKbGPnmN/IUFY7vk+M16LBmzBhMsfN/6YGKs6oL8c= -github.com/projectdiscovery/retryablehttp-go v1.0.20 h1:Ns3m7EPMEFKTSSNPtD1WGkCHvuYyQ6x98HYdKdALqwE= -github.com/projectdiscovery/retryablehttp-go v1.0.20/go.mod h1:3YrxgFe21HUL+25IU9VfFlTZ23yMEA2Zek6p8F55cuI= +github.com/projectdiscovery/retryablehttp-go v1.0.22 h1:GfOELRfB8O2s5LHUwT6gwE95meI9VXaEUpu5T7pGIbc= +github.com/projectdiscovery/retryablehttp-go v1.0.22/go.mod h1:dog12XS0gQZS+nL2SYnzvRJIYFv/ZGDHEtzeKCwkkQc= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8= github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= +github.com/projectdiscovery/tlsx v1.1.4-0.20230822194609-b80759b2fa87 h1:dL7rU5keaHjXh7buBzDLQl5SDlTqJImd5PNETMrzH5Y= +github.com/projectdiscovery/tlsx v1.1.4-0.20230822194609-b80759b2fa87/go.mod h1:g0dTBTD9s3vxsC+JOr0peigKnotxyvpNVwTeRLKlNdQ= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40 h1:bgTXdrA/yFhFGfjhMIsczVNhnsMEHFidgS/FD2Tq5Js= -github.com/projectdiscovery/utils v0.0.45-0.20230725161322-28ec1ee0ba40/go.mod h1:HtUI1pyNCgQUuwZuxDILQ4NSUaFcfBh0TuCK/ZQTS6Q= +github.com/projectdiscovery/utils v0.0.51-0.20230822193513-49253312211f h1:ZhnkkfrvTOQvyzXJOszlntWlMPIXZsEfkJB52sSpdLs= +github.com/projectdiscovery/utils v0.0.51-0.20230822193513-49253312211f/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc= github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= @@ -486,8 +483,8 @@ github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= -github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -643,8 +640,8 @@ golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -677,12 +674,12 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -731,9 +728,9 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -742,7 +739,7 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -754,8 +751,6 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= From 2850daeeba5d95ce43a8548bd613dd7c0cd9907b Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 23 Aug 2023 02:01:30 +0530 Subject: [PATCH 33/38] add documentation in flow package --- v2/pkg/tmplexec/flow/README.md | 137 ++++++++++++++++++++++++-- v2/pkg/tmplexec/flow/flow_internal.go | 35 ------- 2 files changed, 129 insertions(+), 43 deletions(-) diff --git a/v2/pkg/tmplexec/flow/README.md b/v2/pkg/tmplexec/flow/README.md index ba7d1ccdb2..6a14d9c191 100644 --- a/v2/pkg/tmplexec/flow/README.md +++ b/v2/pkg/tmplexec/flow/README.md @@ -110,10 +110,10 @@ Now we can see the template is straight forward and easy to understand. we are f - update variables at runtime (ex: when jwt expires update it by using refresh token and then continue execution) - and a lot more (this is just a tip of iceberg) -orchestration can be understood as nuclei logic bindings for javascript (i.e two way interaction between javascript and nuclei for a specific template) +simply put request execution orchestration can be understood as nuclei logic bindings for javascript (i.e two way interaction between javascript and nuclei for a specific template) To better understand orchestration we can try to build a template for vhost enumeration using flow. which usually requires writing / using a new tool -for simple vhost enumeration we need to +**for simple vhost enumeration we need to** - do a PTR lookup for given ip - get SSL ceritificate for given ip (i.e tls-grab) - extract subject_cn from certificate @@ -187,10 +187,131 @@ In above Js code we are using some Nuclei functions and 1 Map lets understand wh For such complex use case of vhost enumeration just adding 5 lines of js code using nuclei helper functions we achieved vhost enumeration. -**Is this template ready?** -No, we are still missing one thing i.e subject_cn can contain values like `*.projectdiscovery.io` and we need to remove the prefix `*.` -there are lot of ways to do this we can either -- use javscript `replace()` function to remove prefix (ex: `vhost.replace("*.","")`) -- use nuclei js helper function `trimLeft()` (ex: `trimLeft(vhost,"*.")`) -- use dsl helper functions in http request +Final template for vhost along with handling edgecases like +- wildcard prefix in subject_cn,subject_an +- `.` in PTR value +```yaml +id: vhost-enum-flow + +info: + name: vhost enum flow + author: tarunKoyalwar + severity: info + description: | + vhost enumeration by extracting potential vhost names from ssl certificate and dns ptr records + +flow: | + ssl(); + dns({hide: true}); + for (let vhost of iterate(template["ssl_subject_cn"],template["ssl_subject_an"])) { + vhost = vhost.replace("*.", "") + set("vhost", vhost); + http(); + } + +ssl: + - address: "{{Host}}:{{Port}}" + +dns: + - name: "{{FQDN}}" + type: PTR + + matchers: + - type: word + words: + - "IN\tPTR" + + extractors: + - type: regex + name: ptrValue + internal: true + group: 1 + regex: + - "IN\tPTR\t(.+)" + +http: + - raw: + - | + GET / HTTP/1.1 + Host: {{trim_suffix(vhost, ".")}} + + matchers: + - type: status + negative: true + status: + - 400 + - 502 + + extractors: + - type: dsl + dsl: + - '"VHOST: " + vhost + ", SC: " + status_code + ", CL: " + content_length' +``` + + +### Nuclei JS Bindings + +This section contains breif description of all nuclei JS bindings and their usage + +**1. Protocol Execution Functions** + + Any protocol that is present in a nuclei template can be called/executed in javascript in format `proto_name()` i.e `http()` , `dns()` , `ssl()` etc + If we want to execute a specific request of a protocol (ref: see [nuclei-flow-dns](testcases/nuclei-flow-dns-id.yaml)) this can be achieved by either passing + - index of that request in protocol (ex: `dns(0)`, `dns(1)` etc) + - id of that request in protocol (ex: `dns("extract-vps")`, `dns("probe-http")` etc) + For More complex use cases multiple requests of a single protocol can be executed by just specifying their index or id one after another (ex: `dns("extract-vps","1")`) + +**2. Iterate Helper Function** + + Iterate is a nuclei js helper function which can be used to iterate over any type of value (array,map,string,number) while handling empty / nil values. + This is addon helper function to omit boilerplate code of checking if value is empty or not and then iterating over it + ```javascript + iterate(123,{"a":1,"b":2,"c":3}) + // iterate over array with custom separator + iterate([1,2,3,4,5], " ") + ``` + **Note:** In above example we used `iterate(template["ssl_subject_cn"],template["ssl_subject_an"])` which removed lot of boilerplate code of checking if value is empty or not and then iterating over it + +**3. Set Helper Function** + + When Iterating over a values/array or some other use case we might want to invoke a request with custom/given value and this can be achieved by using `set()` helper function. This is acheived by adding this value to template context (global variables) and then using it in request. the format of `set()` is `set("variable_name",value)` ex: `set("username","admin")` etc + ```javascript + for (let vhost of myArray) { + set("vhost", vhost); + http(1) + } + ``` + **Note:** In above example we used `set("vhost", vhost)` which added `vhost` to template context (global variables) and then called `http(1)` which used this value in request + +**4. Template Context** + + when using `nuclei -jsonl` flag we get lot of data/metadata related to a vulnerability (ex: template details,extracted-values and much more) . A template context is nothing but a map/JSON containing all this data along with internal/unexported data that is only available at runtime (ex: extracted values from previous requests, variables added using `set()` etc). This template context is available in javascript as `template` variable and can be used to access any data from it. ex: `template["ssl_subject_cn"]` , `template["ssl_subject_an"]` etc + ```javascript + template["ssl_subject_cn"] // returns value of ssl_subject_cn from template context which is available after executing ssl request + template["ptrValue"] // returns value of ptrValue which was extracted using regex with internal: true + ``` + Lot of times we don't known what all data is available in template context and this can be easily found by printing it to stdout using `log()` function + ```javascript + log(template) + ``` + +**5. Log Helper Function** + + It is a nuclei js alternative to `console.log` and this pretty prints map data in readable format + **Note:** This should be used for debugging purposed only as this prints data to stdout + +**6. Dedupe** + + Lot of times just having arrays/slices is not enough and we might need to remove duplicate variables . for example in earlier vhost enumeration we did not remove any duplicates as there is always a chance of duplicate values in `ssl_subject_cn` and `ssl_subject_an` and this can be achieved by using `dedupe()` object. This is nuclei js helper function to abstract away boilerplate code of removing duplicates from array/slice + ```javascript + let uniq = new Dedupe(); // create new dedupe object + uniq.Add(template["ptrValue"]) + uniq.Add(template["ssl_subject_cn"]); + uniq.Add(template["ssl_subject_an"]); + log(uniq.Values()) + ``` + And that's it , this automatically converts any slice/array to map and removes duplicates from it and returns a slice/array of unique values + +------ +> Similar to DSL helper functions . we can either use built in functions available with `Javscript (ECMAScript 5.1)` or use DSL helper functions and its upto user to decide which one to uses \ No newline at end of file diff --git a/v2/pkg/tmplexec/flow/flow_internal.go b/v2/pkg/tmplexec/flow/flow_internal.go index 6e27fa1619..789f0db174 100644 --- a/v2/pkg/tmplexec/flow/flow_internal.go +++ b/v2/pkg/tmplexec/flow/flow_internal.go @@ -2,7 +2,6 @@ package flow import ( "reflect" - "strings" "sync/atomic" "github.com/dop251/goja" @@ -179,40 +178,6 @@ func (f *FlowExecutor) registerBuiltInFunctions() error { return err } - // unfortunately js doesn't have trimLeft/trimRight - if err := f.jsVM.Set("trimLeft", func(call goja.FunctionCall) goja.Value { - value := call.Argument(0).String() - char := call.Argument(1).String() - if char == "" { - char = " " - } - return f.jsVM.ToValue(strings.TrimLeft(value, char)) - }); err != nil { - return err - } - - if err := f.jsVM.Set("trimRight", func(call goja.FunctionCall) goja.Value { - value := call.Argument(0).String() - char := call.Argument(1).String() - if char == "" { - char = " " - } - return f.jsVM.ToValue(strings.TrimRight(value, char)) - }); err != nil { - return err - } - - if err := f.jsVM.Set("trim", func(call goja.FunctionCall) goja.Value { - value := call.Argument(0).String() - char := call.Argument(1).String() - if char == "" { - char = " " - } - return f.jsVM.ToValue(strings.Trim(value, char)) - }); err != nil { - return err - } - // add a builtin dedupe object if err := f.jsVM.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { d := builtin.NewDedupe(f.jsVM) From 08c33c4aa13dca598c9ee3c5366f2e772cc5a545 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 23 Aug 2023 03:16:31 +0530 Subject: [PATCH 34/38] remove regions.txt file --- v2/cmd/nuclei/regions.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 v2/cmd/nuclei/regions.txt diff --git a/v2/cmd/nuclei/regions.txt b/v2/cmd/nuclei/regions.txt deleted file mode 100644 index dc4c6be0b4..0000000000 --- a/v2/cmd/nuclei/regions.txt +++ /dev/null @@ -1,3 +0,0 @@ -us-east-1 -us-west-1 -ap-south-1 \ No newline at end of file From 20368ed6fb96b126c94f8eca2cf9c3ac994a0471 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 23 Aug 2023 04:03:39 +0530 Subject: [PATCH 35/38] fix minor issue with self contained templates --- v2/pkg/protocols/common/contextargs/metainput.go | 16 ++++++++++++++-- v2/pkg/tmplexec/flow/flow_internal.go | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/v2/pkg/protocols/common/contextargs/metainput.go b/v2/pkg/protocols/common/contextargs/metainput.go index 3c30b82ff8..4027bd7d01 100644 --- a/v2/pkg/protocols/common/contextargs/metainput.go +++ b/v2/pkg/protocols/common/contextargs/metainput.go @@ -15,6 +15,8 @@ type MetaInput struct { Input string `json:"input,omitempty"` // CustomIP to use for connection CustomIP string `json:"customIP,omitempty"` + // hash of the input + hash string `json:"-"` } func (metaInput *MetaInput) marshalToBuffer() (bytes.Buffer, error) { @@ -71,6 +73,16 @@ func (metaInput *MetaInput) PrettyPrint() string { // GetScanHash returns a unique hash that represents a scan by hashing (metainput + templateId) func (metaInput *MetaInput) GetScanHash(templateId string) string { - scanHash := md5.Sum([]byte(templateId + ":" + metaInput.Input + ":" + metaInput.CustomIP)) - return string(scanHash[:]) + // there may be some cases where metainput is changed ex: while executing self-contained template etc + // but that totally changes the scanID/hash so to avoid that we compute hash only once + // and reuse it for all subsequent calls + if metaInput.hash == "" { + metaInput.hash = getMd5Hash(templateId + ":" + metaInput.Input + ":" + metaInput.CustomIP) + } + return metaInput.hash +} + +func getMd5Hash(data string) string { + bin := md5.Sum([]byte(data)) + return string(bin[:]) } diff --git a/v2/pkg/tmplexec/flow/flow_internal.go b/v2/pkg/tmplexec/flow/flow_internal.go index 789f0db174..15765a1476 100644 --- a/v2/pkg/tmplexec/flow/flow_internal.go +++ b/v2/pkg/tmplexec/flow/flow_internal.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/nuclei/v2/pkg/protocols" "github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v2/pkg/tmplexec/flow/builtin" + "github.com/projectdiscovery/nuclei/v2/pkg/types" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -149,7 +150,7 @@ func (f *FlowExecutor) registerBuiltInFunctions() error { if err := f.jsVM.Set("set", func(call goja.FunctionCall) goja.Value { varName := call.Argument(0).Export() varValue := call.Argument(1).Export() - f.options.GetTemplateCtx(f.input.MetaInput).Set(varName.(string), varValue) + f.options.GetTemplateCtx(f.input.MetaInput).Set(types.ToString(varName), varValue) return goja.Null() }); err != nil { return err From f5ac33a8e8cbc92a57c445dcab36bbd29aac8e4e Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Wed, 23 Aug 2023 16:44:52 +0530 Subject: [PATCH 36/38] fix typos of copilot --- v2/pkg/tmplexec/flow/README.md | 48 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/v2/pkg/tmplexec/flow/README.md b/v2/pkg/tmplexec/flow/README.md index 6a14d9c191..0b4fc49814 100644 --- a/v2/pkg/tmplexec/flow/README.md +++ b/v2/pkg/tmplexec/flow/README.md @@ -2,15 +2,14 @@ flow is a new template engine/backend introduced in v3 which primarily adds 2 most awaited features - conditional execution of requests (ex: `flow: dns() && http()`) -- request execution orchestration (iterate over a slice, request execution order) - -both of these features are implemented using javascript (ECMAScript 5.1) and are executed using [goja](https://github.com/dop251/goja) and we can say that flow adds support for request orchestration and conditional execution of requests in templates. +- request execution orchestration (iterate over a slice, request execution order, if/for statement) +both of these features are implemented using javascript (ECMAScript 5.1) with [goja](https://github.com/dop251/goja) backend. ## conditional execution Many times when writing complex templates we might need to add some extra checks (or conditional statements) before executing certain part of request -A idea example of this would be when bruteforcing wordpress login with default usernames and passwords. we can write a new nuclei template for this it would be something like this +An ideal example of this would be when bruteforcing wordpress login with default usernames and passwords. If we try to write a template for this it would be something like this ```yaml id: wordpress-bruteforce info: @@ -41,10 +40,9 @@ http: negative: true ``` -but if we rethink this template, we can see that we are sending 9 requests without even checking first if the url actually exists or target site is actually a wordpress site. before v3 it was possible to do this by adding a extractor and sending additional content in say url fragment and it would fail if request was not successful and another way would be writing a workflow (2 templates and 1 workflow file i.e total 3 files for 1 template) but this is `hacky` and not a good solution. +but if we carefully re-evaluate this template, we can see that template is sending 9 requests without even checking, if the url actually exists or target site is actually a wordpress site. before v3 it was possible to do this by adding a extractor and sending additional content in say url fragment and it would fail if request was not successful and another way would be writing a workflow (2 templates and 1 workflow file i.e total 3 files for 1 template) but this is `hacky` and not a good solution. -With flow in Nuclei v3 we can re-write this template to first check if target is a wordpress site and then execute bruteforce requests. -this can be achieved by doing as simple as `flow: http("check-wp") && http("bruteforce")` +With addition of flow in Nuclei v3 we can re-write this template to first check if target is a wordpress site, if yes then bruteforce login with default credentials and this can be achieved by simply adding one line of content i.e `flow: http("check-wp") && http("bruteforce")` and nuclei will take care of everything else. ```yaml id: wordpress-bruteforce @@ -96,16 +94,16 @@ http: ``` **Note:** this is just a example template with poor matchers. refer 'nuclei-templates' repo for final template -Now we can see the template is straight forward and easy to understand. we are first checking if target is a wordpress site and then executing bruteforce requests. This is just a simple example of conditional execution and flow accepts any Javascript (ECMAScript 5.1) expression/code so you are free to craft any conditional execution logic you want using for , if and whatnot. +The update template now seems straight forward and easy to understand. we are first checking if target is a wordpress site and then executing bruteforce requests. This is just a simple example of conditional execution and flow accepts any Javascript (ECMAScript 5.1) expression/code so you are free to craft any conditional execution logic you want using for,if and whatnot. ## request execution orchestration -`conditional execution` is one simple use case of flow but `flow` is much more powerful than that for example it can be used to +`conditional execution` is one simple use case of flow but `flow` is much more powerful than that, for example it can be used to - iterate over a slice of values and execute requests for each value (ex: [dns-flow-probe](testcases/nuclei-flow-dns.yaml)) -- extract values from one request and iterate over them and execute requests for each value (ex: [[dns-flow-probe](testcases/nuclei-flow-dns.yaml)](https://github.com/projectdiscovery/nuclei/blob/64098b6567a2d6b7fc3e376d61af73836b3277bb/integration_tests/flow/iterate-values-flow.yaml)) +- extract values from one request and iterate over them and execute requests for each value ex: [dns-flow-probe](testcases/nuclei-flow-dns.yaml) - get/set values from/to template context (global variables) - print/log values to stdout at xyz condition or while debugging -- adding custom logic during template execution (ex: if status code is 403 then login and then re-run it) +- adding custom logic during template execution (ex: if status code is 403 => login and then re-run it) - use any/all ECMAScript 5.1 javascript (like objects,arrays etc) and build/transform variables/input at runtime - update variables at runtime (ex: when jwt expires update it by using refresh token and then continue execution) - and a lot more (this is just a tip of iceberg) @@ -113,7 +111,8 @@ Now we can see the template is straight forward and easy to understand. we are f simply put request execution orchestration can be understood as nuclei logic bindings for javascript (i.e two way interaction between javascript and nuclei for a specific template) To better understand orchestration we can try to build a template for vhost enumeration using flow. which usually requires writing / using a new tool -**for simple vhost enumeration we need to** + +**for basic vhost enumeration a template should** - do a PTR lookup for given ip - get SSL ceritificate for given ip (i.e tls-grab) - extract subject_cn from certificate @@ -122,7 +121,7 @@ To better understand orchestration we can try to build a template for vhost enum - and finally bruteforce all found vhosts -**Now if we try to implement this in template it would something like this** +**Now if we try to implement this in template it would be** ```yaml # send a ssl request to get certificate ssl: @@ -166,7 +165,7 @@ http: - '"VHOST: " + vhost + ", SC: " + status_code + ", CL: " + content_length' tarun@macbook:~/Codebase/nuclei/integration_tes ``` **But this template is not yet ready as it is missing core logic i.e how we use all these obtained data and do bruteforce** -and this is where flow comes into picture. it is javascript code with two way bindings to nuclei. if we write javascript code in very simple terms it would be something like this +and this is where flow comes into picture. flow is javascript code with two way bindings to nuclei. if we write javascript code to orchestrate vhost enumeration it is as simple as ```javascript ssl(); dns(); @@ -175,8 +174,10 @@ and this is where flow comes into picture. it is javascript code with two way bi http(); } ``` -With just extra 5 lines of javascript code we can achieve vhost enumeration and run it on scale while also handling filtering of wildcard prefix from vhost values. -In above Js code we are using some Nuclei functions and 1 Map lets understand what they do +With just extra 5 lines of javascript code template can now perform vhost enumeration and this can be run on scale with all awesome features of nuclei with various supported inputs like ASN,CIDR,URL etc + + +In above Js code we are using some Nuclei JS bindings lets understand what they do - `ssl()` => execute ssl request - `dns()` => execute dns request @@ -185,11 +186,12 @@ In above Js code we are using some Nuclei functions and 1 Map lets understand wh - `set("vhost",vhost)` => creates new variable `vhost` in template and assigns value of `vhost` to it - `http()` => execute http request -For such complex use case of vhost enumeration just adding 5 lines of js code using nuclei helper functions we achieved vhost enumeration. -Final template for vhost along with handling edgecases like -- wildcard prefix in subject_cn,subject_an -- `.` in PTR value +This template is now missing one last thing i.e +- removing wildcard prefix (*.) in subject_cn,subject_an +- trailing `.` in PTR value + +and this can be done using either JS methods of using DSL helper functions as shown in below template ```yaml id: vhost-enum-flow @@ -252,7 +254,7 @@ http: ### Nuclei JS Bindings -This section contains breif description of all nuclei JS bindings and their usage +This section contains a brief description of all nuclei JS bindings and their usage **1. Protocol Execution Functions** @@ -265,7 +267,7 @@ This section contains breif description of all nuclei JS bindings and their usag **2. Iterate Helper Function** Iterate is a nuclei js helper function which can be used to iterate over any type of value (array,map,string,number) while handling empty / nil values. - This is addon helper function to omit boilerplate code of checking if value is empty or not and then iterating over it + This is addon helper function from nuclei to omit boilerplate code of checking if value is empty or not and then iterating over it ```javascript iterate(123,{"a":1,"b":2,"c":3}) // iterate over array with custom separator @@ -275,7 +277,7 @@ This section contains breif description of all nuclei JS bindings and their usag **3. Set Helper Function** - When Iterating over a values/array or some other use case we might want to invoke a request with custom/given value and this can be achieved by using `set()` helper function. This is acheived by adding this value to template context (global variables) and then using it in request. the format of `set()` is `set("variable_name",value)` ex: `set("username","admin")` etc + When Iterating over a values/array or some other use case we might want to invoke a request with custom/given value and this can be achieved by using `set()` helper function. When invoked/called it adds given variable to template context (global variables) and that value is used during execution of request/protocol. the format of `set()` is `set("variable_name",value)` ex: `set("username","admin")` etc ```javascript for (let vhost of myArray) { set("vhost", vhost); From 5a38b2f74033953bd7c7216a0a7d82f164928e21 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Mon, 28 Aug 2023 02:12:57 +0530 Subject: [PATCH 37/38] dep + misc update --- v2/go.mod | 19 ++++++++-------- v2/go.sum | 45 ++++++++++++++++++++----------------- v2/pkg/protocols/ssl/ssl.go | 2 +- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 543f63117f..4ced433be3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -21,12 +21,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.19 - github.com/projectdiscovery/fastdialer v0.0.35 - github.com/projectdiscovery/hmap v0.0.13 + github.com/projectdiscovery/fastdialer v0.0.36 + github.com/projectdiscovery/hmap v0.0.15 github.com/projectdiscovery/interactsh v1.1.4 github.com/projectdiscovery/rawhttp v0.1.18 - github.com/projectdiscovery/retryabledns v1.0.32 - github.com/projectdiscovery/retryablehttp-go v1.0.22 + github.com/projectdiscovery/retryabledns v1.0.35 + github.com/projectdiscovery/retryablehttp-go v1.0.24 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -69,7 +69,7 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/projectdiscovery/dsl v0.0.16 github.com/projectdiscovery/fasttemplate v0.0.2 - github.com/projectdiscovery/goflags v0.1.17 + github.com/projectdiscovery/goflags v0.1.18 github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 github.com/projectdiscovery/httpx v1.3.4 @@ -77,9 +77,9 @@ require ( github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.4-0.20230822194609-b80759b2fa87 + github.com/projectdiscovery/tlsx v1.1.4 github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 - github.com/projectdiscovery/utils v0.0.51-0.20230822193513-49253312211f + github.com/projectdiscovery/utils v0.0.51 github.com/projectdiscovery/wappalyzergo v0.0.107 github.com/stretchr/testify v1.8.4 gopkg.in/src-d/go-git.v4 v4.13.1 @@ -115,7 +115,7 @@ require ( github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kataras/jwt v0.1.8 // indirect @@ -131,7 +131,8 @@ require ( github.com/projectdiscovery/asnmap v1.0.4 // indirect github.com/projectdiscovery/cdncheck v1.0.9 // indirect github.com/projectdiscovery/freeport v0.0.5 // indirect - github.com/refraction-networking/utls v1.3.2 // indirect + github.com/quic-go/quic-go v0.37.0 // indirect + github.com/refraction-networking/utls v1.4.2 // indirect github.com/sashabaranov/go-openai v1.14.1 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skeema/knownhosts v1.1.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index 9370c2180e..efaa4f96f8 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -189,6 +189,7 @@ github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw4 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE= github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -202,6 +203,7 @@ github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y= github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/goburrow/cache v0.1.4 h1:As4KzO3hgmzPlnaMniZU9+VmoNYseUhuELbxy9mRBfw= github.com/goburrow/cache v0.1.4/go.mod h1:cDFesZDnIlrHoNlMYqqMpCRawuXulgx+y7mXU8HZ+/c= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -265,8 +267,8 @@ github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUD github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf h1:umfGUaWdFP2s6457fz1+xXYIWDxdGc7HdkLS9aJ1skk= github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf/go.mod h1:V99KdStnMHZsvVOwIvhfcUzYgYkRZeQWUtumtL+SKxA= github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= @@ -388,8 +390,9 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= @@ -415,20 +418,20 @@ github.com/projectdiscovery/clistats v0.0.19 h1:SA/qRHbmS9VEbVEPzX/ka01hZDYATL9Z github.com/projectdiscovery/clistats v0.0.19/go.mod h1:NQDAW/O7cK9xBIgk46kJjwGRkjSg5JkB8E4DvuxXr+c= github.com/projectdiscovery/dsl v0.0.16 h1:ECymBWfB6L6M/y0X6fa+mwg2l0nCSUkfoJkesjGCYJ4= github.com/projectdiscovery/dsl v0.0.16/go.mod h1:OiVbde6xGMM4NXnf3DUJIEqdwWppPADBSPMrxDHwRCU= -github.com/projectdiscovery/fastdialer v0.0.35 h1:dCjYaZ2dOtKmIbQ7OUuf/pZiMQRHfUjjLoHrEF8CJ8g= -github.com/projectdiscovery/fastdialer v0.0.35/go.mod h1:dTx0C7JRWKKO5ZxGqM0NUDzB4svmyYqGM6zcHIk2ueo= +github.com/projectdiscovery/fastdialer v0.0.36 h1:Ac/CRLryJB2mA8erDwAHoCJGFvjCDIPUznxWl9kJPW8= +github.com/projectdiscovery/fastdialer v0.0.36/go.mod h1:jxX9iQJdTwlD6u0Q9Dj9/AmatHPW2GRl3V6XTAvKtHY= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= -github.com/projectdiscovery/goflags v0.1.17 h1:Fr+28b7y5K3/aiSfN97X7jTtk0XcGgMKaT8I2X+7ZYI= -github.com/projectdiscovery/goflags v0.1.17/go.mod h1:n0TWe/lpb7AZCNagDRX9VPKB1gY+/o+EfD2zd8lJJJk= +github.com/projectdiscovery/goflags v0.1.18 h1:L4nwDBNJcZhbmhI3GhQ1GJwz7xVWFL3BumJ+TIDBi5E= +github.com/projectdiscovery/goflags v0.1.18/go.mod h1:cZut0Q98yksNVtM73RPSm22N/eDkAMFT9t6mwu6S5pY= github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6 h1:M74WAoZ99q/LJPHC8aIWIt8+FLh699KqLm2CUSHoytA= github.com/projectdiscovery/gozero v0.0.0-20230510004414-f1d11fdaf5c6/go.mod h1:jCpXNvLUCPMzm5AhJv8wtnUt/7rz0TY2SsqvKQ8tn2E= -github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI= -github.com/projectdiscovery/hmap v0.0.13/go.mod h1:Ymc9xjbfhswpmI/gOx5hyR4+OvqguSq1SDJTH197gWg= +github.com/projectdiscovery/hmap v0.0.15 h1:iTRXV94bNIuR5obYBxOhvs3yEYXdNdJJnrXnxv4uLHc= +github.com/projectdiscovery/hmap v0.0.15/go.mod h1:oybodscUwBbL4GnhBPPTemazPXyMErqL+dE+0ZtJ6lg= github.com/projectdiscovery/httpx v1.3.4 h1:1tCP7YRngCDi2a8PvvcYqmpR1H9X7Qgn89uazKL65eg= github.com/projectdiscovery/httpx v1.3.4/go.mod h1:5JlNJcEHPF9ByFFNEcaXEAs8yZYsUC6E9Q3VGfDpPeY= github.com/projectdiscovery/interactsh v1.1.4 h1:1qVxJ14aG/X7TLJoK5AHnaX6I7hnbPp5R2ql1bSYzqI= @@ -443,27 +446,27 @@ github.com/projectdiscovery/rawhttp v0.1.18 h1:wTs6CePrjcIz5/SrxkluOrCGOk3F9Ddt3 github.com/projectdiscovery/rawhttp v0.1.18/go.mod h1:nwTySMnfI7qFMQEC9PHdklXGWED8FDcEOnA8DGZqu/A= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.32 h1:Ekr+1j1jwQ2qINW7T02uMcXFc3QeduN3vOligpfQgeo= -github.com/projectdiscovery/retryabledns v1.0.32/go.mod h1:t8aKbGPnmN/IUFY7vk+M16LBmzBhMsfN/6YGKs6oL8c= -github.com/projectdiscovery/retryablehttp-go v1.0.22 h1:GfOELRfB8O2s5LHUwT6gwE95meI9VXaEUpu5T7pGIbc= -github.com/projectdiscovery/retryablehttp-go v1.0.22/go.mod h1:dog12XS0gQZS+nL2SYnzvRJIYFv/ZGDHEtzeKCwkkQc= +github.com/projectdiscovery/retryabledns v1.0.35 h1:lPX8f7exDaiNJc/4Rc44xQfFK9BpA8ZLtpQ+te2ymLU= +github.com/projectdiscovery/retryabledns v1.0.35/go.mod h1:V4nRoHJzK2UmlGgKMRduLBkgNNMXJXmJchB5Wui8s4c= +github.com/projectdiscovery/retryablehttp-go v1.0.24 h1:1In7vIUnNvEdHhnA5KmUVf+D+tVZgITaJUZxFYgKCdo= +github.com/projectdiscovery/retryablehttp-go v1.0.24/go.mod h1:S2KiViUrjvRua/mifKEj+6Gs8TJjhFsrZwOXKJAZzSA= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/tlsx v1.1.1 h1:4q14vu2A+TnQjhYI68I3yCUss3UM0fmrkmnJKqoYRQ8= -github.com/projectdiscovery/tlsx v1.1.1/go.mod h1:x2S3KajTVxH5Tm4lbBoX4EumY/gh+cGzfBUhlCuNtdY= -github.com/projectdiscovery/tlsx v1.1.4-0.20230822194609-b80759b2fa87 h1:dL7rU5keaHjXh7buBzDLQl5SDlTqJImd5PNETMrzH5Y= -github.com/projectdiscovery/tlsx v1.1.4-0.20230822194609-b80759b2fa87/go.mod h1:g0dTBTD9s3vxsC+JOr0peigKnotxyvpNVwTeRLKlNdQ= +github.com/projectdiscovery/tlsx v1.1.4 h1:jXRvichO/ZfhYERch1CbNS1PRbS2KgSBj7JoWslEpIw= +github.com/projectdiscovery/tlsx v1.1.4/go.mod h1:crzMlxOokVQDwGVm51JPZi1ZAgzxhNl1KVRmbff6pkI= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1 h1:Pu6LvDqn+iSlhCDKKWm1ItPc++kqqlU8OntZeB/Prak= github.com/projectdiscovery/uncover v1.0.6-0.20230601103158-bfd7e02a5bb1/go.mod h1:Drl/CWD392mKtdXJhCBPlMkM0I6671pqedFphcnK5f8= -github.com/projectdiscovery/utils v0.0.51-0.20230822193513-49253312211f h1:ZhnkkfrvTOQvyzXJOszlntWlMPIXZsEfkJB52sSpdLs= -github.com/projectdiscovery/utils v0.0.51-0.20230822193513-49253312211f/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc= +github.com/projectdiscovery/utils v0.0.51 h1:WZV8kP4VW/I1ZkXDjyoudhhfVVHpAKKnW+Re0LVNMbc= +github.com/projectdiscovery/utils v0.0.51/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc= github.com/projectdiscovery/wappalyzergo v0.0.107 h1:B8gzJpAh08f1o+OiDunHAfKtqXiDnFCc7Rj1qKp+DB8= github.com/projectdiscovery/wappalyzergo v0.0.107/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= -github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= -github.com/refraction-networking/utls v1.3.2/go.mod h1:fmoaOww2bxzzEpIKOebIsnBvjQpqP7L2vcm/9KUfm/E= +github.com/quic-go/quic-go v0.37.0 h1:wf/Ym2yeWi98oQn4ahiBSqdnaXVxNQGj2oBQFgiVChc= +github.com/quic-go/quic-go v0.37.0/go.mod h1:XtCUOCALTTWbPyd0IxFfHf6h0sEMubRFvEYHl3QxKw8= +github.com/refraction-networking/utls v1.4.2 h1:7N+928mSM1pEyAJb8x2Y1FbEwTIftGwn2IFykosSzwc= +github.com/refraction-networking/utls v1.4.2/go.mod h1:JkUIj+Pc8eyFB0z+A4RJRZmoT43ajjFZWVMXuZQ8BEQ= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= diff --git a/v2/pkg/protocols/ssl/ssl.go b/v2/pkg/protocols/ssl/ssl.go index 90b0563f95..dd8b1faf3f 100644 --- a/v2/pkg/protocols/ssl/ssl.go +++ b/v2/pkg/protocols/ssl/ssl.go @@ -222,7 +222,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } requestOptions.Output.Request(requestOptions.TemplateID, hostPort, request.Type().String(), err) - gologger.Verbose().Msgf("Sent SSL request to %s", hostPort) + gologger.Verbose().Msgf("[%s] Sent SSL request to %s", request.options.TemplateID, hostPort) if requestOptions.Options.Debug || requestOptions.Options.DebugRequests || requestOptions.Options.StoreResponse { msg := fmt.Sprintf("[%s] Dumped SSL request for %s", requestOptions.TemplateID, input.MetaInput.Input) From 4f079d98d5599c91efe79840c48d9a6c5f054686 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 29 Aug 2023 00:52:10 +0530 Subject: [PATCH 38/38] fix reqID: use req.Type instead of template.Type --- v2/pkg/templates/templates.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/v2/pkg/templates/templates.go b/v2/pkg/templates/templates.go index eb613b9eea..72a3cb069b 100644 --- a/v2/pkg/templates/templates.go +++ b/v2/pkg/templates/templates.go @@ -187,35 +187,35 @@ func (template *Template) validateAllRequestIDs() { if len(template.RequestsCode) > 1 { for i, req := range template.RequestsCode { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } } if len(template.RequestsDNS) > 1 { for i, req := range template.RequestsDNS { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } } if len(template.RequestsFile) > 1 { for i, req := range template.RequestsFile { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } } if len(template.RequestsHTTP) > 1 { for i, req := range template.RequestsHTTP { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } } if len(template.RequestsHeadless) > 1 { for i, req := range template.RequestsHeadless { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } @@ -223,28 +223,28 @@ func (template *Template) validateAllRequestIDs() { if len(template.RequestsNetwork) > 1 { for i, req := range template.RequestsNetwork { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } } if len(template.RequestsSSL) > 1 { for i, req := range template.RequestsSSL { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } } if len(template.RequestsWebsocket) > 1 { for i, req := range template.RequestsWebsocket { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } } if len(template.RequestsWHOIS) > 1 { for i, req := range template.RequestsWHOIS { if req.ID == "" { - req.ID = template.Type().String() + "_" + strconv.Itoa(i) + req.ID = req.Type().String() + "_" + strconv.Itoa(i) } } }