From 09ba28623e11cb071d604754d20620aebafcd73c Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 2 Oct 2023 22:14:00 +0530 Subject: [PATCH 1/2] conditional debug + profiling --- v2/cmd/nuclei/main_debug.go | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 v2/cmd/nuclei/main_debug.go diff --git a/v2/cmd/nuclei/main_debug.go b/v2/cmd/nuclei/main_debug.go new file mode 100644 index 0000000000..18e2423643 --- /dev/null +++ b/v2/cmd/nuclei/main_debug.go @@ -0,0 +1,69 @@ +//go:build debug +// +build debug + +package main + +import ( + "log" + "os" + "runtime" + "runtime/pprof" + "strconv" + "time" +) + +func init() { + memfile := os.Getenv("PPROF_FILE") + cpuProfile := os.Getenv("CPU_PROFILE") + if cpuProfile == "" { + cpuProfile = "cpuprofile.out" + } + if memfile == "" { + memfile = "memdump.out" + } + log.Printf("GOOS: %v\n", runtime.GOOS) + log.Printf("GOARCH: %v\n", runtime.GOARCH) + + pproftime := os.Getenv("PPROF_TIME") // in seconds + + var ticker *time.Ticker + if pproftime != "" { + d, err := time.ParseDuration(pproftime) + if err == nil { + ticker = time.NewTicker(d) + log.Printf("profile: tick every %v\n", d.String()) + } + } + if ticker == nil { + ticker = time.NewTicker(1 * time.Second) // tick every second + log.Println("profile: tick every second") + } + f, err := os.Create(memfile) + if err != nil { + log.Fatalf("profile: could not create memory profile %q: %v", memProfile, err) + } + cpuf, err := os.Create(cpuProfile) + if err != nil { + log.Fatalf("profile: could not create cpu profile %q: %v", cpuProfile, err) + } + pprof.StartCPUProfile(cpuf) + mempprofRate := 4096 + tmp := os.Getenv("MEM_PROFILE_RATE") + if tmp != "" { + tmpint, err := strconv.Atoi(tmp) + if err == nil { + mempprofRate = tmpint + } + } + runtime.MemProfileRate = mempprofRate + log.Printf("profile: memory profiling enabled (rate %d), %s\n", runtime.MemProfileRate, memProfile) + + go func() { + for { + <-ticker.C + pprof.Lookup("heap").WriteTo(f, 0) + pprof.StopCPUProfile() + pprof.StartCPUProfile(cpuf) + } + }() +} From ae0da1a9f2d34d63f5971a8a443fbbd96a182ee1 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Tue, 3 Oct 2023 01:00:29 +0530 Subject: [PATCH 2/2] donot concatenate profiles --- v2/cmd/nuclei/main_debug.go | 82 +++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 26 deletions(-) diff --git a/v2/cmd/nuclei/main_debug.go b/v2/cmd/nuclei/main_debug.go index 18e2423643..e535fbbfcd 100644 --- a/v2/cmd/nuclei/main_debug.go +++ b/v2/cmd/nuclei/main_debug.go @@ -4,8 +4,10 @@ package main import ( + "bytes" "log" "os" + "path/filepath" "runtime" "runtime/pprof" "strconv" @@ -13,40 +15,29 @@ import ( ) func init() { - memfile := os.Getenv("PPROF_FILE") - cpuProfile := os.Getenv("CPU_PROFILE") + log.Println("available env configs:") + log.Println("MEM_PROFILE_DIR - directory to write memory profiles to") + log.Println("CPU_PROFILE_DIR - directory to write cpu profiles to") + log.Println("PPROF_TIME - polling time for cpu and memory profiles (with unit ex: 10s)") + log.Println("MEM_PROFILE_RATE - memory profiling rate (default 4096)") + memfile := os.Getenv("MEM_PROFILE_DIR") + cpuProfile := os.Getenv("CPU_PROFILE_DIR") + pproftime := os.Getenv("PPROF_TIME") if cpuProfile == "" { - cpuProfile = "cpuprofile.out" + cpuProfile = "cpuprofile" } if memfile == "" { - memfile = "memdump.out" + memfile = "memdump" } - log.Printf("GOOS: %v\n", runtime.GOOS) - log.Printf("GOARCH: %v\n", runtime.GOARCH) - - pproftime := os.Getenv("PPROF_TIME") // in seconds - var ticker *time.Ticker + tickerTime := time.Duration(3 * time.Second) if pproftime != "" { d, err := time.ParseDuration(pproftime) if err == nil { - ticker = time.NewTicker(d) - log.Printf("profile: tick every %v\n", d.String()) + tickerTime = d } } - if ticker == nil { - ticker = time.NewTicker(1 * time.Second) // tick every second - log.Println("profile: tick every second") - } - f, err := os.Create(memfile) - if err != nil { - log.Fatalf("profile: could not create memory profile %q: %v", memProfile, err) - } - cpuf, err := os.Create(cpuProfile) - if err != nil { - log.Fatalf("profile: could not create cpu profile %q: %v", cpuProfile, err) - } - pprof.StartCPUProfile(cpuf) + mempprofRate := 4096 tmp := os.Getenv("MEM_PROFILE_RATE") if tmp != "" { @@ -55,15 +46,54 @@ func init() { mempprofRate = tmpint } } + + log.Printf("GOOS: %v\n", runtime.GOOS) + log.Printf("GOARCH: %v\n", runtime.GOARCH) + + _ = os.MkdirAll(memfile, 0755) + _ = os.MkdirAll(cpuProfile, 0755) + runtime.MemProfileRate = mempprofRate log.Printf("profile: memory profiling enabled (rate %d), %s\n", runtime.MemProfileRate, memProfile) + log.Printf("profile: ticker enabled (rate %s)\n", tickerTime) + // cpu ticker and profiler go func() { + ticker := time.NewTicker(tickerTime) + count := 0 + buff := bytes.Buffer{} + log.Printf("profile: cpu profiling enabled (ticker %s)\n", tickerTime) for { + err := pprof.StartCPUProfile(&buff) + if err != nil { + log.Fatalf("profile: could not start cpu profile: %s\n", err) + } <-ticker.C - pprof.Lookup("heap").WriteTo(f, 0) pprof.StopCPUProfile() - pprof.StartCPUProfile(cpuf) + if err := os.WriteFile(filepath.Join(cpuProfile, "cpuprofile-t"+strconv.Itoa(count)+".out"), buff.Bytes(), 0755); err != nil { + log.Fatalf("profile: could not write cpu profile: %s\n", err) + } + buff.Reset() + count++ + } + }() + + // memory ticker and profiler + go func() { + ticker := time.NewTicker(tickerTime) + count := 0 + log.Printf("profile: memory profiling enabled (ticker %s)\n", tickerTime) + for { + <-ticker.C + var buff bytes.Buffer + if err := pprof.WriteHeapProfile(&buff); err != nil { + log.Printf("profile: could not write memory profile: %s\n", err) + } + err := os.WriteFile(filepath.ToSlash(filepath.Join(memfile, "memprofile-t"+strconv.Itoa(count)+".out")), buff.Bytes(), 0755) + if err != nil { + log.Printf("profile: could not write memory profile: %s\n", err) + } + count++ } }() }