From fb5a7134f2290133a83b40a3ae19e3fe29509bf5 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 26 Apr 2024 23:06:51 +0530 Subject: [PATCH 01/15] feat: fixed leak --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 35a39ccc08..c3114de23f 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.91 + github.com/projectdiscovery/utils v0.0.92-0.20240426173301-f79c675670b0 github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index ce3f2b214d..e778343670 100644 --- a/go.sum +++ b/go.sum @@ -343,7 +343,6 @@ github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwSmwnCg4bxg= github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= -github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -891,6 +890,8 @@ github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLP github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= github.com/projectdiscovery/utils v0.0.91 h1:aHAAnC0qX9pJZrWq4Qpl2PSTYLrSCL1dm1QWLjprE2w= github.com/projectdiscovery/utils v0.0.91/go.mod h1:O/6U3ZoU+tNw4lKurdjyVMZPVXL5IYq0YeaDc15PRls= +github.com/projectdiscovery/utils v0.0.92-0.20240426173301-f79c675670b0 h1:GPzoTwDQhFLtz/EIFDmUv21/VVgR7o7VtVO92VVnAxE= +github.com/projectdiscovery/utils v0.0.92-0.20240426173301-f79c675670b0/go.mod h1:X2jJZ83pGfIp2adwGdXTG2jKIYAVcTXpRpHzB8PD+OM= github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= From 5a6cbd437cd7dbfa5761e5aed911987a5970266c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Sat, 27 Apr 2024 00:20:21 +0530 Subject: [PATCH 02/15] add go leak unit test in sdk --- go.mod | 1 + go.sum | 8 ++++---- lib/{ => tests}/sdk_test.go | 10 +++++++++- 3 files changed, 14 insertions(+), 5 deletions(-) rename lib/{ => tests}/sdk_test.go (91%) diff --git a/go.mod b/go.mod index c3114de23f..bb092ce862 100644 --- a/go.mod +++ b/go.mod @@ -100,6 +100,7 @@ require ( github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 + go.uber.org/goleak v1.3.0 golang.org/x/term v0.19.0 gopkg.in/yaml.v3 v3.0.1 moul.io/http2curl v1.0.0 diff --git a/go.sum b/go.sum index e778343670..f2eb7283fd 100644 --- a/go.sum +++ b/go.sum @@ -317,6 +317,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= @@ -888,8 +890,6 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= -github.com/projectdiscovery/utils v0.0.91 h1:aHAAnC0qX9pJZrWq4Qpl2PSTYLrSCL1dm1QWLjprE2w= -github.com/projectdiscovery/utils v0.0.91/go.mod h1:O/6U3ZoU+tNw4lKurdjyVMZPVXL5IYq0YeaDc15PRls= github.com/projectdiscovery/utils v0.0.92-0.20240426173301-f79c675670b0 h1:GPzoTwDQhFLtz/EIFDmUv21/VVgR7o7VtVO92VVnAxE= github.com/projectdiscovery/utils v0.0.92-0.20240426173301-f79c675670b0/go.mod h1:X2jJZ83pGfIp2adwGdXTG2jKIYAVcTXpRpHzB8PD+OM= github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= @@ -1151,8 +1151,8 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= diff --git a/lib/sdk_test.go b/lib/tests/sdk_test.go similarity index 91% rename from lib/sdk_test.go rename to lib/tests/sdk_test.go index 710b2a6ed7..3c9963522a 100644 --- a/lib/sdk_test.go +++ b/lib/tests/sdk_test.go @@ -1,13 +1,15 @@ -package nuclei_test +package sdk_test import ( "testing" nuclei "github.com/projectdiscovery/nuclei/v3/lib" "github.com/stretchr/testify/require" + "go.uber.org/goleak" ) func TestSimpleNuclei(t *testing.T) { + defer goleak.VerifyNone(t) ne, err := nuclei.NewNucleiEngine( nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), @@ -21,6 +23,7 @@ func TestSimpleNuclei(t *testing.T) { } func TestSimpleNucleiRemote(t *testing.T) { + defer goleak.VerifyNone(t) ne, err := nuclei.NewNucleiEngine( nuclei.WithTemplatesOrWorkflows( nuclei.TemplateSources{ @@ -39,6 +42,7 @@ func TestSimpleNucleiRemote(t *testing.T) { } func TestThreadSafeNuclei(t *testing.T) { + defer goleak.VerifyNone(t) // create nuclei engine with options ne, err := nuclei.NewThreadSafeNucleiEngine() require.Nil(t, err) @@ -58,3 +62,7 @@ func TestThreadSafeNuclei(t *testing.T) { // wait for all scans to finish defer ne.Close() } + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} From 55b877e1f44c0cdddc903f3be2a3b6ffce105ed0 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Sat, 27 Apr 2024 03:40:15 +0530 Subject: [PATCH 03/15] added goleak unit tests --- go.mod | 4 +- go.sum | 4 ++ lib/sdk_private.go | 9 ++- lib/tests/sdk_test.go | 134 +++++++++++++++++++++++++++--------------- 4 files changed, 101 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index bb092ce862..761aea6099 100644 --- a/go.mod +++ b/go.mod @@ -99,8 +99,8 @@ require ( github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 + github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 - go.uber.org/goleak v1.3.0 golang.org/x/term v0.19.0 gopkg.in/yaml.v3 v3.0.1 moul.io/http2curl v1.0.0 @@ -180,6 +180,7 @@ require ( github.com/klauspost/compress v1.17.6 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/logrusorgru/aurora/v4 v4.0.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mackerelio/go-osstat v0.2.4 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -233,6 +234,7 @@ require ( github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect + go.uber.org/goleak v1.3.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/sync v0.6.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect diff --git a/go.sum b/go.sum index f2eb7283fd..533aad655d 100644 --- a/go.sum +++ b/go.sum @@ -673,6 +673,8 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= +github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3 h1:wIONC+HMNRqmWBjuMxhatuSzHaljStc4gjDeKycxy0A= github.com/lor00x/goldap v0.0.0-20180618054307-a546dffdd1a3/go.mod h1:37YR9jabpiIxsb8X9VCIx8qFOjTDIIrIHHODa8C4gz0= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -1017,6 +1019,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc h1:/5P5I7oDqdLee8W9Moof0xSD8tT1qEVzhObSI9CqHkg= +github.com/tarunKoyalwar/goleak v0.0.0-20240426214851-746d64600adc/go.mod h1:uQdBQGrE1fZ2EyOs0pLcCDd1bBV4rSThieuIIGhXZ50= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= diff --git a/lib/sdk_private.go b/lib/sdk_private.go index 89628cc933..13e8746510 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -35,6 +35,8 @@ import ( "github.com/projectdiscovery/ratelimit" ) +var sharedInit sync.Once = sync.Once{} + // applyRequiredDefaults to options func (e *NucleiEngine) applyRequiredDefaults() { mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate) @@ -116,8 +118,11 @@ func (e *NucleiEngine) init() error { e.parser = templates.NewParser() - _ = protocolstate.Init(e.opts) - _ = protocolinit.Init(e.opts) + sharedInit.Do(func() { + _ = protocolstate.Init(e.opts) + _ = protocolinit.Init(e.opts) + }) + e.applyRequiredDefaults() var err error diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go index 3c9963522a..8d1fff3016 100644 --- a/lib/tests/sdk_test.go +++ b/lib/tests/sdk_test.go @@ -1,68 +1,108 @@ package sdk_test import ( + "os" + "os/exec" "testing" nuclei "github.com/projectdiscovery/nuclei/v3/lib" + "github.com/projectdiscovery/utils/env" "github.com/stretchr/testify/require" - "go.uber.org/goleak" + "github.com/tarunKoyalwar/goleak" ) func TestSimpleNuclei(t *testing.T) { - defer goleak.VerifyNone(t) - ne, err := nuclei.NewNucleiEngine( - nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), - nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), - ) - require.Nil(t, err) - ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here - // when callback is nil it nuclei will print JSON output to stdout - err = ne.ExecuteWithCallback(nil) - require.Nil(t, err) - defer ne.Close() + fn := func() { + defer goleak.VerifyNone(t, goleak.Pretty()) + ne, err := nuclei.NewNucleiEngine( + nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), + nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), + ) + require.Nil(t, err) + ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here + // when callback is nil it nuclei will print JSON output to stdout + err = ne.ExecuteWithCallback(nil) + require.Nil(t, err) + defer ne.Close() + } + + // this is shared test so needs to be run as seperate process + if env.GetEnvOrDefault("TestSimpleNuclei", false) { + // run as new process + cmd := exec.Command(os.Args[0], "-test.run=TestSimpleNuclei") + cmd.Env = append(os.Environ(), "TestSimpleNuclei=true") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("process ran with error %s, output: %s", err, out) + } + } else { + fn() + } } func TestSimpleNucleiRemote(t *testing.T) { - defer goleak.VerifyNone(t) - ne, err := nuclei.NewNucleiEngine( - nuclei.WithTemplatesOrWorkflows( - nuclei.TemplateSources{ - RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"}, - }, - ), - ) - require.Nil(t, err) - ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here - err = ne.LoadAllTemplates() - require.Nil(t, err, "could not load templates") - // when callback is nil it nuclei will print JSON output to stdout - err = ne.ExecuteWithCallback(nil) - require.Nil(t, err) - defer ne.Close() + fn := func() { + defer goleak.VerifyNone(t, goleak.Pretty()) + ne, err := nuclei.NewNucleiEngine( + nuclei.WithTemplatesOrWorkflows( + nuclei.TemplateSources{ + RemoteTemplates: []string{"https://cloud.projectdiscovery.io/public/nameserver-fingerprint.yaml"}, + }, + ), + ) + require.Nil(t, err) + ne.LoadTargets([]string{"scanme.sh"}, false) // probe non http/https target is set to false here + err = ne.LoadAllTemplates() + require.Nil(t, err, "could not load templates") + // when callback is nil it nuclei will print JSON output to stdout + err = ne.ExecuteWithCallback(nil) + require.Nil(t, err) + defer ne.Close() + } + // this is shared test so needs to be run as seperate process + if env.GetEnvOrDefault("TestSimpleNucleiRemote", false) { + cmd := exec.Command(os.Args[0], "-test.run=TestSimpleNucleiRemote") + cmd.Env = append(os.Environ(), "TestSimpleNucleiRemote=true") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("process ran with error %s, output: %s", err, out) + } + } else { + fn() + } } func TestThreadSafeNuclei(t *testing.T) { - defer goleak.VerifyNone(t) - // create nuclei engine with options - ne, err := nuclei.NewThreadSafeNucleiEngine() - require.Nil(t, err) - - // scan 1 = run dns templates on scanme.sh - t.Run("scanme.sh", func(t *testing.T) { - err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + fn := func() { + defer goleak.VerifyNone(t, goleak.Pretty()) + // create nuclei engine with options + ne, err := nuclei.NewThreadSafeNucleiEngine() require.Nil(t, err) - }) - // scan 2 = run dns templates on honey.scanme.sh - t.Run("honey.scanme.sh", func(t *testing.T) { - err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) - require.Nil(t, err) - }) + // scan 1 = run dns templates on scanme.sh + t.Run("scanme.sh", func(t *testing.T) { + err = ne.ExecuteNucleiWithOpts([]string{"scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + require.Nil(t, err) + }) - // wait for all scans to finish - defer ne.Close() -} + // scan 2 = run dns templates on honey.scanme.sh + t.Run("honey.scanme.sh", func(t *testing.T) { + err = ne.ExecuteNucleiWithOpts([]string{"honey.scanme.sh"}, nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"})) + require.Nil(t, err) + }) + + // wait for all scans to finish + defer ne.Close() + } -func TestMain(m *testing.M) { - goleak.VerifyTestMain(m) + if env.GetEnvOrDefault("TestThreadSafeNuclei", false) { + cmd := exec.Command(os.Args[0], "-test.run=TestThreadSafeNuclei") + cmd.Env = append(os.Environ(), "TestThreadSafeNuclei=true") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("process ran with error %s, output: %s", err, out) + } + } else { + fn() + } } From ea26124ddff99d7e12e5f115c2496eddb98f5244 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sat, 27 Apr 2024 21:25:10 +0530 Subject: [PATCH 04/15] bugfix: add random user agents to fuzzing requests --- pkg/input/types/http.go | 5 +++++ pkg/protocols/http/request_fuzz.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/pkg/input/types/http.go b/pkg/input/types/http.go index a407d6a066..dbc5c9e325 100644 --- a/pkg/input/types/http.go +++ b/pkg/input/types/http.go @@ -12,6 +12,7 @@ import ( "sync" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/useragent" "github.com/projectdiscovery/utils/conversion" mapsutil "github.com/projectdiscovery/utils/maps" urlutil "github.com/projectdiscovery/utils/url" @@ -73,6 +74,10 @@ func (rr *RequestResponse) BuildRequest() (*retryablehttp.Request, error) { req.Header.Add(k, v) return true }) + if req.Header.Get("User-Agent") == "" { + userAgent := useragent.PickRandom() + req.Header.Set("User-Agent", userAgent.Raw) + } rr.req = req }) return rr.req, rr.reqErr diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index fc2a4e7576..42065576ad 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -24,6 +24,7 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/retryablehttp-go" + "github.com/projectdiscovery/useragent" urlutil "github.com/projectdiscovery/utils/url" ) @@ -89,6 +90,9 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, previous if err != nil { return errors.Wrap(err, "fuzz: could not build request from url") } + userAgent := useragent.PickRandom() + baseRequest.Header.Set("User-Agent", userAgent.Raw) + // execute with one value first to checks its applicability err = request.executeAllFuzzingRules(inputx, previous, baseRequest, callback) if err != nil { From 65d40fdd73e6ac371626d040fba8123b3dbe8a19 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sun, 28 Apr 2024 01:54:36 +0530 Subject: [PATCH 05/15] misc --- go.mod | 4 +- go.sum | 2 - pkg/core/executors.go | 53 +------------------ pkg/js/compiler/non-pool.go | 2 +- .../common/automaticscan/automaticscan.go | 3 -- .../common/protocolstate/memguardian.go | 6 ++- 6 files changed, 9 insertions(+), 61 deletions(-) diff --git a/go.mod b/go.mod index 761aea6099..e4a71358a3 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/projectdiscovery/nuclei/v3 go 1.21 +replace github.com/projectdiscovery/utils => /Users/ice3man/hack/tt/utils + require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/andygrunwald/go-jira v1.16.0 @@ -94,7 +96,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.92-0.20240426173301-f79c675670b0 + github.com/projectdiscovery/utils v0.0.92-0.20240427201850-f32137d057ad github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index 533aad655d..3b76b482b0 100644 --- a/go.sum +++ b/go.sum @@ -892,8 +892,6 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= -github.com/projectdiscovery/utils v0.0.92-0.20240426173301-f79c675670b0 h1:GPzoTwDQhFLtz/EIFDmUv21/VVgR7o7VtVO92VVnAxE= -github.com/projectdiscovery/utils v0.0.92-0.20240426173301-f79c675670b0/go.mod h1:X2jJZ83pGfIp2adwGdXTG2jKIYAVcTXpRpHzB8PD+OM= github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= diff --git a/pkg/core/executors.go b/pkg/core/executors.go index 89c85b2ad7..3cf6a2ade1 100644 --- a/pkg/core/executors.go +++ b/pkg/core/executors.go @@ -162,7 +162,7 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t // wp is workpool that contains different waitgroups for // headless and non-headless templates // global waitgroup should not be used here - wp := e.GetWorkPool() + wp := e.WorkPool() for _, tpl := range alltemplates { select { @@ -212,54 +212,3 @@ func (e *Engine) executeTemplatesOnTarget(ctx context.Context, alltemplates []*t } wp.Wait() } - -type ChildExecuter struct { - e *Engine - - results *atomic.Bool -} - -// Close closes the executer returning bool results -func (e *ChildExecuter) Close() *atomic.Bool { - e.e.workPool.Wait() - return e.results -} - -// Execute executes a template and URLs -func (e *ChildExecuter) Execute(template *templates.Template, value *contextargs.MetaInput) { - templateType := template.Type() - - // resize check point - nop if there are no changes - e.e.WorkPool().RefreshWithConfig(e.e.GetWorkPoolConfig()) - - var wg *syncutil.AdaptiveWaitGroup - if templateType == types.HeadlessProtocol { - wg = e.e.workPool.Headless - } else { - wg = e.e.workPool.Default - } - - wg.Add() - go func(tpl *templates.Template) { - defer wg.Done() - - // TODO: Workflows are a no-op for now. We need to - // implement them in the future with context cancellation - ctxArgs := contextargs.New(context.Background()) - ctxArgs.MetaInput = value - ctx := scan.NewScanContext(context.Background(), ctxArgs) - match, err := template.Executer.Execute(ctx) - if err != nil { - gologger.Warning().Msgf("[%s] Could not execute step: %s\n", e.e.executerOpts.Colorizer.BrightBlue(template.ID), err) - } - e.results.CompareAndSwap(false, match) - }(template) -} - -// ExecuteWithOpts executes with the full options -func (e *Engine) ChildExecuter() *ChildExecuter { - return &ChildExecuter{ - e: e, - results: &atomic.Bool{}, - } -} diff --git a/pkg/js/compiler/non-pool.go b/pkg/js/compiler/non-pool.go index 218b89b82a..74c0230357 100644 --- a/pkg/js/compiler/non-pool.go +++ b/pkg/js/compiler/non-pool.go @@ -8,7 +8,7 @@ import ( ) var ( - ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency)) + ephemeraljsc *syncutil.AdaptiveWaitGroup lazyFixedSgInit = sync.OnceFunc(func() { ephemeraljsc, _ = syncutil.New(syncutil.WithSize(NonPoolingVMConcurrency)) }) diff --git a/pkg/protocols/common/automaticscan/automaticscan.go b/pkg/protocols/common/automaticscan/automaticscan.go index 4567f498d9..2a7988cb04 100644 --- a/pkg/protocols/common/automaticscan/automaticscan.go +++ b/pkg/protocols/common/automaticscan/automaticscan.go @@ -56,7 +56,6 @@ type Service struct { engine *core.Engine target provider.InputProvider wappalyzer *wappalyzer.Wappalyze - childExecuter *core.ChildExecuter httpclient *retryablehttp.Client templateDirs []string // root Template Directories technologyMappings map[string]string @@ -95,7 +94,6 @@ func New(opts Options) (*Service, error) { return nil, err } - childExecuter := opts.Engine.ChildExecuter() httpclient, err := httpclientpool.Get(opts.ExecuterOpts.Options, &httpclientpool.Configuration{ Connection: &httpclientpool.ConnectionConfiguration{ DisableKeepAlive: httputil.ShouldDisableKeepAlive(opts.ExecuterOpts.Options), @@ -111,7 +109,6 @@ func New(opts Options) (*Service, error) { target: opts.Target, wappalyzer: wappalyzer, templateDirs: templateDirs, // fix this - childExecuter: childExecuter, httpclient: httpclient, technologyMappings: mappingData, techTemplates: techDetectTemplates, diff --git a/pkg/protocols/common/protocolstate/memguardian.go b/pkg/protocols/common/protocolstate/memguardian.go index f70269d060..6a9d1e297a 100644 --- a/pkg/protocols/common/protocolstate/memguardian.go +++ b/pkg/protocols/common/protocolstate/memguardian.go @@ -21,7 +21,7 @@ func StartActiveMemGuardian() { return } - memTimer := time.NewTicker(memguardian.DefaultInterval) + memTimer = time.NewTicker(memguardian.DefaultInterval) go func() { for range memTimer.C { if IsLowOnMemory() { @@ -38,7 +38,9 @@ func StopActiveMemGuardian() { return } - memTimer.Stop() + if memTimer != nil { + memTimer.Stop() + } } func IsLowOnMemory() bool { From 08c325ceab536b69126bab2267d678583bb5ec9a Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sun, 28 Apr 2024 02:05:24 +0530 Subject: [PATCH 06/15] misc --- go.mod | 2 -- go.sum | 3 --- .../common/protocolstate/memguardian.go | 20 ++++++++++++++----- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index e4a71358a3..87a18c2f15 100644 --- a/go.mod +++ b/go.mod @@ -145,8 +145,6 @@ require ( github.com/docker/cli v24.0.5+incompatible // indirect github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect - github.com/eapache/channels v1.1.0 // indirect - github.com/eapache/queue v1.1.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect diff --git a/go.sum b/go.sum index 3b76b482b0..1f524a13b0 100644 --- a/go.sum +++ b/go.sum @@ -296,11 +296,8 @@ github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k= -github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= diff --git a/pkg/protocols/common/protocolstate/memguardian.go b/pkg/protocols/common/protocolstate/memguardian.go index 6a9d1e297a..f30d6fca77 100644 --- a/pkg/protocols/common/protocolstate/memguardian.go +++ b/pkg/protocols/common/protocolstate/memguardian.go @@ -1,6 +1,7 @@ package protocolstate import ( + "context" "sync" "time" @@ -14,6 +15,7 @@ var ( MaxThreadsOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_THREADS", 0) MaxBytesBufferAllocOnLowMemory = env.GetEnvOrDefault("MEMGUARDIAN_ALLOC", 0) memTimer *time.Ticker + cancelFunc context.CancelFunc ) func StartActiveMemGuardian() { @@ -22,12 +24,19 @@ func StartActiveMemGuardian() { } memTimer = time.NewTicker(memguardian.DefaultInterval) + var ctx context.Context + ctx, cancelFunc = context.WithCancel(context.Background()) go func() { - for range memTimer.C { - if IsLowOnMemory() { - _ = GlobalGuardBytesBufferAlloc() - } else { - GlobalRestoreBytesBufferAlloc() + for { + select { + case <-ctx.Done(): + return + case <-memTimer.C: + if IsLowOnMemory() { + _ = GlobalGuardBytesBufferAlloc() + } else { + GlobalRestoreBytesBufferAlloc() + } } } }() @@ -40,6 +49,7 @@ func StopActiveMemGuardian() { if memTimer != nil { memTimer.Stop() + cancelFunc() } } From 29ae84c2c6ff28a2157ce0d3f4646fceea946b4c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 29 Apr 2024 18:31:15 +0530 Subject: [PATCH 07/15] fix lint + use utils pr + misc --- go.mod | 4 +--- go.sum | 2 ++ internal/runner/inputs.go | 5 ++++- pkg/core/workpool.go | 15 +++++++++++---- pkg/js/compiler/pool.go | 9 ++++++--- pkg/protocols/dns/request.go | 4 +++- pkg/protocols/http/httputils/spm.go | 5 +++-- pkg/protocols/http/request.go | 4 +++- pkg/protocols/javascript/js.go | 4 +++- pkg/protocols/network/request.go | 6 +++++- 10 files changed, 41 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 87a18c2f15..24ac6aafcc 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/projectdiscovery/nuclei/v3 go 1.21 -replace github.com/projectdiscovery/utils => /Users/ice3man/hack/tt/utils - require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/andygrunwald/go-jira v1.16.0 @@ -96,7 +94,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.92-0.20240427201850-f32137d057ad + github.com/projectdiscovery/utils v0.0.92-0.20240429121553-f9776e1f163a github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index 1f524a13b0..468ccfc946 100644 --- a/go.sum +++ b/go.sum @@ -889,6 +889,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= +github.com/projectdiscovery/utils v0.0.92-0.20240429121553-f9776e1f163a h1:XisN6GYdG6VlaTV5Gz5u7njxAowmaUUBq/KiRPC31Yw= +github.com/projectdiscovery/utils v0.0.92-0.20240429121553-f9776e1f163a/go.mod h1:fs4mAOGTdyP0yoDywJcWHodDHFI8itWyiMzlAapJFmQ= github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= diff --git a/internal/runner/inputs.go b/internal/runner/inputs.go index 047360aee3..4ead9946e6 100644 --- a/internal/runner/inputs.go +++ b/internal/runner/inputs.go @@ -1,6 +1,7 @@ package runner import ( + "context" "sync/atomic" "time" @@ -50,7 +51,9 @@ func (r *Runner) initializeTemplatesHTTPInput() (*hybrid.HybridMap, error) { } if r.options.ProbeConcurrency > 0 && swg.Size != r.options.ProbeConcurrency { - swg.Resize(r.options.ProbeConcurrency) + if err := swg.Resize(context.Background(), r.options.ProbeConcurrency); err != nil { + gologger.Error().Msgf("Could not resize workpool: %s\n", err) + } } swg.Add() diff --git a/pkg/core/workpool.go b/pkg/core/workpool.go index 810f99392d..f3773933ce 100644 --- a/pkg/core/workpool.go +++ b/pkg/core/workpool.go @@ -1,6 +1,9 @@ package core import ( + "context" + + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/templates/types" syncutil "github.com/projectdiscovery/utils/sync" ) @@ -71,14 +74,18 @@ func (w *WorkPool) RefreshWithConfig(config WorkPoolConfig) { if w.config.HeadlessInputConcurrency != config.HeadlessInputConcurrency { w.config.HeadlessInputConcurrency = config.HeadlessInputConcurrency } - w.Refresh() + w.Refresh(context.Background()) } -func (w *WorkPool) Refresh() { +func (w *WorkPool) Refresh(ctx context.Context) { if w.Default.Size != w.config.TypeConcurrency { - w.Default.Resize(w.config.TypeConcurrency) + if err := w.Default.Resize(ctx, w.config.TypeConcurrency); err != nil { + gologger.Warning().Msgf("Could not resize workpool: %s\n", err) + } } if w.Headless.Size != w.config.HeadlessTypeConcurrency { - w.Headless.Resize(w.config.HeadlessTypeConcurrency) + if err := w.Headless.Resize(ctx, w.config.HeadlessTypeConcurrency); err != nil { + gologger.Warning().Msgf("Could not resize workpool: %s\n", err) + } } } diff --git a/pkg/js/compiler/pool.go b/pkg/js/compiler/pool.go index 678b9afb0a..17d9f746f5 100644 --- a/pkg/js/compiler/pool.go +++ b/pkg/js/compiler/pool.go @@ -2,6 +2,7 @@ package compiler import ( "bytes" + "context" "encoding/json" "fmt" "reflect" @@ -55,10 +56,12 @@ var ( lazySgInit = sync.OnceFunc(func() { pooljsc, _ = syncutil.New(syncutil.WithSize(PoolingJsVmConcurrency)) }) - sgResizeCheck = func() { + sgResizeCheck = func(ctx context.Context) { // resize check point if pooljsc.Size != PoolingJsVmConcurrency { - pooljsc.Resize(PoolingJsVmConcurrency) + if err := pooljsc.Resize(ctx, PoolingJsVmConcurrency); err != nil { + gologger.Warning().Msgf("Could not resize workpool: %s\n", err) + } } } ) @@ -122,7 +125,7 @@ func executeWithPoolingProgram(p *goja.Program, args *ExecuteArgs, opts *Execute // its unknown (most likely cannot be done) to limit max js runtimes at a moment without making it static // unlike sync.Pool which reacts to GC and its purposes is to reuse objects rather than creating new ones lazySgInit() - sgResizeCheck() + sgResizeCheck(opts.Context) pooljsc.Add() defer pooljsc.Done() diff --git a/pkg/protocols/dns/request.go b/pkg/protocols/dns/request.go index 608e4730ac..b4f04118e5 100644 --- a/pkg/protocols/dns/request.go +++ b/pkg/protocols/dns/request.go @@ -88,7 +88,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, // resize check point - nop if there are no changes if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency { - swg.Resize(request.options.Options.PayloadConcurrency) + if err := swg.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil { + return err + } } value = generators.MergeMaps(vars, value) diff --git a/pkg/protocols/http/httputils/spm.go b/pkg/protocols/http/httputils/spm.go index 5e20fea58d..bb0b337c77 100644 --- a/pkg/protocols/http/httputils/spm.go +++ b/pkg/protocols/http/httputils/spm.go @@ -181,10 +181,11 @@ func (h *StopAtFirstMatchHandler[T]) Release() { } } -func (h *StopAtFirstMatchHandler[T]) Resize(size int) { +func (h *StopAtFirstMatchHandler[T]) Resize(ctx context.Context, size int) error { if h.sgPool.Size != size { - h.sgPool.Resize(size) + return h.sgPool.Resize(ctx, size) } + return nil } func (h *StopAtFirstMatchHandler[T]) Size() int { diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index b2628f007a..df914a785c 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -243,7 +243,9 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV // resize check point - nop if there are no changes if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency { - spmHandler.Resize(request.options.Options.PayloadConcurrency) + if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil { + return err + } } // break if stop at first match is found or host is unresponsive diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 61257a968b..84457b5fca 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -434,7 +434,9 @@ func (request *Request) executeRequestParallel(ctxParent context.Context, hostPo // resize check point - nop if there are no changes if shouldFollowGlobal && sg.Size != request.options.Options.PayloadConcurrency { - sg.Resize(request.options.Options.PayloadConcurrency) + if err := sg.Resize(ctxParent, request.options.Options.PayloadConcurrency); err != nil { + gologger.Warning().Msgf("Could not resize workpool: %s\n", err) + } } sg.Add() diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 23eed8cb0f..1123f40569 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -200,7 +200,11 @@ func (request *Request) executeAddress(variables map[string]interface{}, actualA // resize check point - nop if there are no changes if shouldFollowGlobal && swg.Size != request.options.Options.PayloadConcurrency { - swg.Resize(request.options.Options.PayloadConcurrency) + if err := swg.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil { + m.Lock() + multiErr = multierr.Append(multiErr, err) + m.Unlock() + } } value = generators.MergeMaps(value, payloads) From c9fbbe89ac3d4209c960bb1a350f964af29f1651 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 29 Apr 2024 18:41:03 +0530 Subject: [PATCH 08/15] fix ratelimit memleak in sdk --- lib/multi.go | 6 ++++++ lib/sdk.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/lib/multi.go b/lib/multi.go index bdcee79677..05a3b80372 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -112,6 +112,12 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts .. if err != nil { return err } + // cleanup and stop all resources + defer func() { + if unsafeOpts.executerOpts.RateLimiter != nil { + unsafeOpts.executerOpts.RateLimiter.Stop() + } + }() // load templates workflowLoader, err := workflow.NewLoader(&unsafeOpts.executerOpts) diff --git a/lib/sdk.go b/lib/sdk.go index f6b5ea7c43..5efb6ca73b 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -189,6 +189,9 @@ func (e *NucleiEngine) Close() { e.customWriter.Close() e.hostErrCache.Close() e.executerOpts.RateLimiter.Stop() + if e.rateLimiter != nil { + e.rateLimiter.Stop() + } } // ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) From 20e08959adf4ee900e1c573f4328f9bb63a2469c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 29 Apr 2024 18:58:13 +0530 Subject: [PATCH 09/15] close protocolstate shared resources in nuclei sdk/lib --- lib/sdk.go | 3 +++ pkg/protocols/common/protocolstate/memguardian.go | 5 ++--- pkg/protocols/common/protocolstate/state.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/sdk.go b/lib/sdk.go index 5efb6ca73b..3961b3da4c 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" "github.com/projectdiscovery/nuclei/v3/pkg/reporting" "github.com/projectdiscovery/nuclei/v3/pkg/templates" @@ -192,6 +193,8 @@ func (e *NucleiEngine) Close() { if e.rateLimiter != nil { e.rateLimiter.Stop() } + // close global shared resources + protocolstate.Close() } // ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) diff --git a/pkg/protocols/common/protocolstate/memguardian.go b/pkg/protocols/common/protocolstate/memguardian.go index f30d6fca77..1db1e0a775 100644 --- a/pkg/protocols/common/protocolstate/memguardian.go +++ b/pkg/protocols/common/protocolstate/memguardian.go @@ -18,14 +18,13 @@ var ( cancelFunc context.CancelFunc ) -func StartActiveMemGuardian() { +func StartActiveMemGuardian(ctx context.Context) { if memguardian.DefaultMemGuardian == nil { return } memTimer = time.NewTicker(memguardian.DefaultInterval) - var ctx context.Context - ctx, cancelFunc = context.WithCancel(context.Background()) + ctx, cancelFunc = context.WithCancel(ctx) go func() { for { select { diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index 3298c91366..67820ec074 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -148,7 +148,7 @@ func Init(options *types.Options) error { return Dialer.Dial(ctx, "tcp", addr) }) - StartActiveMemGuardian() + StartActiveMemGuardian(context.Background()) return nil } From ffc57bfeceff1aa7f4c7be758091d024b4c1bfd8 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 29 Apr 2024 19:21:54 +0530 Subject: [PATCH 10/15] add missing close references --- lib/multi.go | 23 ++++++++++++++++++----- lib/sdk.go | 32 +++++++++++++++++++++++++++----- pkg/input/types/probe.go | 2 ++ pkg/utils/http_probe.go | 7 +++++++ 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/lib/multi.go b/lib/multi.go index 05a3b80372..57336a7f7c 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -58,6 +58,22 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt return u, nil } +// closeEphemeralObjects closes all resources used by ephemeral nuclei objects/instances/types +func closeEphemeralObjects(u *unsafeOptions) { + if u.executerOpts.RateLimiter != nil { + u.executerOpts.RateLimiter.Stop() + } + // dereference all objects that were inherited from base nuclei engine + // since these are meant to be closed globally by base nuclei engine + u.executerOpts.Output = nil + u.executerOpts.IssuesClient = nil + u.executerOpts.Interactsh = nil + u.executerOpts.HostErrorsCache = nil + u.executerOpts.Progress = nil + u.executerOpts.Catalog = nil + u.executerOpts.Parser = nil +} + // ThreadSafeNucleiEngine is a tweaked version of nuclei.Engine whose methods are thread-safe // and can be used concurrently. Non-thread-safe methods start with Global prefix type ThreadSafeNucleiEngine struct { @@ -107,17 +123,14 @@ func (e *ThreadSafeNucleiEngine) ExecuteNucleiWithOpts(targets []string, opts .. return err } } + defer tmpEngine.Close() // create ephemeral nuclei objects/instances/types using base nuclei engine unsafeOpts, err := createEphemeralObjects(e.eng, tmpEngine.opts) if err != nil { return err } // cleanup and stop all resources - defer func() { - if unsafeOpts.executerOpts.RateLimiter != nil { - unsafeOpts.executerOpts.RateLimiter.Stop() - } - }() + defer closeEphemeralObjects(unsafeOpts) // load templates workflowLoader, err := workflow.NewLoader(&unsafeOpts.executerOpts) diff --git a/lib/sdk.go b/lib/sdk.go index 3961b3da4c..7959d4c1d5 100644 --- a/lib/sdk.go +++ b/lib/sdk.go @@ -185,16 +185,38 @@ func (e *NucleiEngine) SignTemplate(tmplSigner *signer.TemplateSigner, data []by // Close all resources used by nuclei engine func (e *NucleiEngine) Close() { - e.interactshClient.Close() - e.rc.Close() - e.customWriter.Close() - e.hostErrCache.Close() - e.executerOpts.RateLimiter.Stop() + if e.interactshClient != nil { + e.interactshClient.Close() + } + if e.rc != nil { + e.rc.Close() + } + if e.customWriter != nil { + e.customWriter.Close() + } + if e.customProgress != nil { + e.customProgress.Stop() + } + if e.hostErrCache != nil { + e.hostErrCache.Close() + } + if e.executerOpts.RateLimiter != nil { + e.executerOpts.RateLimiter.Stop() + } if e.rateLimiter != nil { e.rateLimiter.Stop() } // close global shared resources protocolstate.Close() + if e.inputProvider != nil { + e.inputProvider.Close() + } + if e.browserInstance != nil { + e.browserInstance.Close() + } + if e.httpxClient != nil { + _ = e.httpxClient.Close() + } } // ExecuteWithCallback executes templates on targets and calls callback on each result(only if results are found) diff --git a/pkg/input/types/probe.go b/pkg/input/types/probe.go index 006faada94..c34adead20 100644 --- a/pkg/input/types/probe.go +++ b/pkg/input/types/probe.go @@ -4,4 +4,6 @@ package types type InputLivenessProbe interface { // ProbeURL probes the scheme for a URL. first HTTPS is tried ProbeURL(input string) (string, error) + // Close closes the liveness probe + Close() error } diff --git a/pkg/utils/http_probe.go b/pkg/utils/http_probe.go index 19059b2af9..1ee4657039 100644 --- a/pkg/utils/http_probe.go +++ b/pkg/utils/http_probe.go @@ -43,6 +43,13 @@ func (i *inputLivenessChecker) ProbeURL(input string) (string, error) { return ProbeURL(input, i.client), nil } +func (i *inputLivenessChecker) Close() error { + if i.client.Dialer != nil { + i.client.Dialer.Close() + } + return nil +} + // GetInputLivenessChecker returns a new input liveness checker using provided httpx client func GetInputLivenessChecker(client *httpx.HTTPX) types.InputLivenessProbe { x := &inputLivenessChecker{client: client} From a1c0a573cea2d19b8c6f0077f95f9edcffd8137c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 29 Apr 2024 19:34:30 +0530 Subject: [PATCH 11/15] ignore read/write loop of intransit connections --- lib/tests/sdk_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go index 8d1fff3016..80e2780f7a 100644 --- a/lib/tests/sdk_test.go +++ b/lib/tests/sdk_test.go @@ -11,9 +11,17 @@ import ( "github.com/tarunKoyalwar/goleak" ) +var knownLeaks = []goleak.Option{ + // prettyify the output and generate dependency graph and more details instead of just stack output + goleak.Pretty(), + // this is not a leak but idle http connection that is not closed yet by transport + goleak.IgnoreAnyFunction("net/http.(*persistConn).readLoop"), + goleak.IgnoreAnyFunction("net/http.(*persistConn).writeLoop"), +} + func TestSimpleNuclei(t *testing.T) { fn := func() { - defer goleak.VerifyNone(t, goleak.Pretty()) + defer goleak.VerifyNone(t, knownLeaks...) ne, err := nuclei.NewNucleiEngine( nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), @@ -42,7 +50,7 @@ func TestSimpleNuclei(t *testing.T) { func TestSimpleNucleiRemote(t *testing.T) { fn := func() { - defer goleak.VerifyNone(t, goleak.Pretty()) + defer goleak.VerifyNone(t, knownLeaks...) ne, err := nuclei.NewNucleiEngine( nuclei.WithTemplatesOrWorkflows( nuclei.TemplateSources{ @@ -74,7 +82,7 @@ func TestSimpleNucleiRemote(t *testing.T) { func TestThreadSafeNuclei(t *testing.T) { fn := func() { - defer goleak.VerifyNone(t, goleak.Pretty()) + defer goleak.VerifyNone(t, knownLeaks...) // create nuclei engine with options ne, err := nuclei.NewThreadSafeNucleiEngine() require.Nil(t, err) From f31e6a8f62280a8134f5b0f521569653db8c4818 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 29 Apr 2024 21:29:41 +0530 Subject: [PATCH 12/15] close unnecessary idle conns --- lib/tests/sdk_test.go | 25 +++++++++++++++++++------ pkg/installer/versioncheck.go | 4 ++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go index 80e2780f7a..698d72932f 100644 --- a/lib/tests/sdk_test.go +++ b/lib/tests/sdk_test.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "testing" + "time" nuclei "github.com/projectdiscovery/nuclei/v3/lib" "github.com/projectdiscovery/utils/env" @@ -14,14 +15,16 @@ import ( var knownLeaks = []goleak.Option{ // prettyify the output and generate dependency graph and more details instead of just stack output goleak.Pretty(), - // this is not a leak but idle http connection that is not closed yet by transport - goleak.IgnoreAnyFunction("net/http.(*persistConn).readLoop"), - goleak.IgnoreAnyFunction("net/http.(*persistConn).writeLoop"), } func TestSimpleNuclei(t *testing.T) { fn := func() { - defer goleak.VerifyNone(t, knownLeaks...) + defer func() { + // resources like leveldb have a delay to commit in-memory resources + // to disk, typically 1-2 seconds, so we wait for 2 seconds + time.Sleep(2 * time.Second) + goleak.VerifyNone(t, knownLeaks...) + }() ne, err := nuclei.NewNucleiEngine( nuclei.WithTemplateFilters(nuclei.TemplateFilters{ProtocolTypes: "dns"}), nuclei.EnableStatsWithOpts(nuclei.StatsOptions{JSON: true}), @@ -50,7 +53,12 @@ func TestSimpleNuclei(t *testing.T) { func TestSimpleNucleiRemote(t *testing.T) { fn := func() { - defer goleak.VerifyNone(t, knownLeaks...) + defer func() { + // resources like leveldb have a delay to commit in-memory resources + // to disk, typically 1-2 seconds, so we wait for 2 seconds + time.Sleep(2 * time.Second) + goleak.VerifyNone(t, knownLeaks...) + }() ne, err := nuclei.NewNucleiEngine( nuclei.WithTemplatesOrWorkflows( nuclei.TemplateSources{ @@ -82,7 +90,12 @@ func TestSimpleNucleiRemote(t *testing.T) { func TestThreadSafeNuclei(t *testing.T) { fn := func() { - defer goleak.VerifyNone(t, knownLeaks...) + defer func() { + // resources like leveldb have a delay to commit in-memory resources + // to disk, typically 1-2 seconds, so we wait for 2 seconds + time.Sleep(2 * time.Second) + goleak.VerifyNone(t, knownLeaks...) + }() // create nuclei engine with options ne, err := nuclei.NewThreadSafeNucleiEngine() require.Nil(t, err) diff --git a/pkg/installer/versioncheck.go b/pkg/installer/versioncheck.go index d8e1ae5251..7e2e82d6ff 100644 --- a/pkg/installer/versioncheck.go +++ b/pkg/installer/versioncheck.go @@ -83,6 +83,10 @@ func UpdateIgnoreFile() error { } func doVersionCheck(isSDK bool) error { + // we use global retryablehttp client so its not immeditely gc'd if any references are held + // and according our config we have idle connections which are shown as leaked by goleak in tests + // i.e we close all idle connections after our use and it doesn't affect any other part of the code + defer retryableHttpClient.HTTPClient.CloseIdleConnections() resp, err := retryableHttpClient.Get(pdtmNucleiVersionEndpoint + "?" + getpdtmParams(isSDK)) if err != nil { return err From 57e4efcbc09f14a87d2ac6ab5670c9d3379519a2 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 29 Apr 2024 21:37:14 +0530 Subject: [PATCH 13/15] add ignore method --- lib/tests/sdk_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/tests/sdk_test.go b/lib/tests/sdk_test.go index 698d72932f..ee688d797e 100644 --- a/lib/tests/sdk_test.go +++ b/lib/tests/sdk_test.go @@ -15,6 +15,9 @@ import ( var knownLeaks = []goleak.Option{ // prettyify the output and generate dependency graph and more details instead of just stack output goleak.Pretty(), + // net/http transport maintains idle connections which are closed with cooldown + // hence they don't count as leaks + goleak.IgnoreAnyFunction("net/http.(*http2ClientConn).readLoop"), } func TestSimpleNuclei(t *testing.T) { From 03e3840891fafd90ba771fe45741aea75c4e83c6 Mon Sep 17 00:00:00 2001 From: mzack Date: Mon, 29 Apr 2024 21:07:06 +0200 Subject: [PATCH 14/15] using fixed utils --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 24ac6aafcc..c9cbe22e42 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.92-0.20240429121553-f9776e1f163a + github.com/projectdiscovery/utils v0.0.92-0.20240429190444-f4bcedb988b5 github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index 468ccfc946..66c5fb0e3e 100644 --- a/go.sum +++ b/go.sum @@ -891,6 +891,8 @@ github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLP github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= github.com/projectdiscovery/utils v0.0.92-0.20240429121553-f9776e1f163a h1:XisN6GYdG6VlaTV5Gz5u7njxAowmaUUBq/KiRPC31Yw= github.com/projectdiscovery/utils v0.0.92-0.20240429121553-f9776e1f163a/go.mod h1:fs4mAOGTdyP0yoDywJcWHodDHFI8itWyiMzlAapJFmQ= +github.com/projectdiscovery/utils v0.0.92-0.20240429190444-f4bcedb988b5 h1:wKORc3Mh/EaaCi4l/dI6Rfc6YBWVicSZg/Blhhlt8SY= +github.com/projectdiscovery/utils v0.0.92-0.20240429190444-f4bcedb988b5/go.mod h1:fs4mAOGTdyP0yoDywJcWHodDHFI8itWyiMzlAapJFmQ= github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= From 042c656bf607fd5b2766529f8ec7ce82214cecb8 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Tue, 30 Apr 2024 18:28:19 +0530 Subject: [PATCH 15/15] dep update --- go.mod | 6 +++--- go.sum | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index c9cbe22e42..e5f58fc0df 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.68 + github.com/projectdiscovery/fastdialer v0.0.69 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/interactsh v1.1.9 github.com/projectdiscovery/rawhttp v0.1.45 github.com/projectdiscovery/retryabledns v1.0.58 - github.com/projectdiscovery/retryablehttp-go v1.0.57 + github.com/projectdiscovery/retryablehttp-go v1.0.58 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -94,7 +94,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.47 - github.com/projectdiscovery/utils v0.0.92-0.20240429190444-f4bcedb988b5 + github.com/projectdiscovery/utils v0.0.92 github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 diff --git a/go.sum b/go.sum index 66c5fb0e3e..47a2aa352b 100644 --- a/go.sum +++ b/go.sum @@ -837,8 +837,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.0.52 h1:jvIvF+qN8+MbI1MHtWJJKfWqAZQlCExL3ob7SddQbZE= github.com/projectdiscovery/dsl v0.0.52/go.mod h1:xfcHwhy2HSaeGgh+1wqzOoCGm2XTdh5JzjBRBVHEMvI= -github.com/projectdiscovery/fastdialer v0.0.68 h1:JuIrr8aVGdGWkEwL4axsJWAWDY2uviSqBB0TCekeCOo= -github.com/projectdiscovery/fastdialer v0.0.68/go.mod h1:asHSBFJgmwrXpiegcrcAgOyd/QewCVgeI4idH55+v7M= +github.com/projectdiscovery/fastdialer v0.0.69 h1:BfFQTyTB1hrw9sWCw4CjQfbmlpvnJCPZEmKtxcwJGbU= +github.com/projectdiscovery/fastdialer v0.0.69/go.mod h1:RXrx7M2T3V3rMZ2h0X2/SsY93+RhgF/LmFa1E0MI3L8= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -877,8 +877,8 @@ github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gB github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.57 h1:OGfUXKXV4bE5msGxeRrNtMaDg2l8U1JcLXmwG7yXWrY= -github.com/projectdiscovery/retryablehttp-go v1.0.57/go.mod h1:Lo2EU1wV1draQ/dHuiSkokW4gZ216F/qi/t12DIdMbA= +github.com/projectdiscovery/retryablehttp-go v1.0.58 h1:i5BlSJGgNnoTULyqLSe3d39o/XShxK4oEvx0e/gb9N4= +github.com/projectdiscovery/retryablehttp-go v1.0.58/go.mod h1:bbok7sSEplSwZOY91UlLdVilhavYos1RaCJLJx761V0= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -889,10 +889,8 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.47 h1:VEOU7uG7TutZNIE0DZNP7hGAGi4bwLPGM1X7Rny52s0= github.com/projectdiscovery/useragent v0.0.47/go.mod h1:Cfk9X9SISYSCmqpej0r9+paJbDHzNHic2YdWQtpdz2M= -github.com/projectdiscovery/utils v0.0.92-0.20240429121553-f9776e1f163a h1:XisN6GYdG6VlaTV5Gz5u7njxAowmaUUBq/KiRPC31Yw= -github.com/projectdiscovery/utils v0.0.92-0.20240429121553-f9776e1f163a/go.mod h1:fs4mAOGTdyP0yoDywJcWHodDHFI8itWyiMzlAapJFmQ= -github.com/projectdiscovery/utils v0.0.92-0.20240429190444-f4bcedb988b5 h1:wKORc3Mh/EaaCi4l/dI6Rfc6YBWVicSZg/Blhhlt8SY= -github.com/projectdiscovery/utils v0.0.92-0.20240429190444-f4bcedb988b5/go.mod h1:fs4mAOGTdyP0yoDywJcWHodDHFI8itWyiMzlAapJFmQ= +github.com/projectdiscovery/utils v0.0.92 h1:lGCmjUJhzoNX4FQZWpp80058pRlD0/dYxLJOSs07EqY= +github.com/projectdiscovery/utils v0.0.92/go.mod h1:d5uvD5qcRiK3qxZbBy9eatCqrCSuj9SObL04w/WgXSg= github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE=