From 0177cefe8a75b14b007e8b11d9d6e1452bbd09eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Geisend=C3=B6rfer?= Date: Sun, 10 Nov 2024 15:37:54 +0100 Subject: [PATCH] coredump: Add gosym sub command gosym symbolizes go test cases using addr2line. This is useful for debugging the correctness of go stack unwinding results. --- tools/coredump/coredump.go | 38 ++++++- tools/coredump/gosym.go | 206 +++++++++++++++++++++++++++++++++++++ tools/coredump/json.go | 9 +- tools/coredump/main.go | 1 + 4 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 tools/coredump/gosym.go diff --git a/tools/coredump/coredump.go b/tools/coredump/coredump.go index eff7b37c..c3506d79 100644 --- a/tools/coredump/coredump.go +++ b/tools/coredump/coredump.go @@ -10,6 +10,8 @@ import ( "fmt" "os" "runtime" + "strconv" + "strings" "time" "unsafe" @@ -114,16 +116,44 @@ func (c *symbolizationCache) symbolize(ty libpf.FrameType, fileID libpf.FileID, } if data, ok := c.symbols[libpf.NewFrameID(fileID, lineNumber)]; ok { - return fmt.Sprintf("%s+%d in %s:%d", - data.FunctionName, data.FunctionOffset, - data.SourceFile, data.SourceLine), nil + return formatSymbolizedFrame(data, true), nil } sourceFile, ok := c.files[fileID] if !ok { sourceFile = fmt.Sprintf("%08x", fileID) } - return fmt.Sprintf("%s+0x%x", sourceFile, lineNumber), nil + return formatUnsymbolizedFrame(sourceFile, lineNumber), nil +} + +func formatSymbolizedFrame(frame *reporter.FrameMetadataArgs, functionOffsets bool) string { + var funcOffset string + if functionOffsets { + funcOffset = "+" + strconv.Itoa(int(frame.FunctionOffset)) + } + return fmt.Sprintf("%s%s in %s:%d", + frame.FunctionName, funcOffset, + frame.SourceFile, frame.SourceLine) +} + +func formatUnsymbolizedFrame(file string, addr libpf.AddressOrLineno) string { + return fmt.Sprintf("%s+0x%x", file, addr) +} + +func parseUnsymbolizedFrame(frame string) (file string, addr libpf.AddressOrLineno, err error) { + fileS, addrS, found := strings.Cut(frame, "+0x") + if !found { + err = fmt.Errorf("bad frame string: %q", frame) + return + } + file = fileS + var addrU uint64 + addrU, err = strconv.ParseUint(addrS, 16, 64) + if err != nil { + return + } + addr = libpf.AddressOrLineno(addrU) + return } func ExtractTraces(ctx context.Context, pr process.Process, debug bool, diff --git a/tools/coredump/gosym.go b/tools/coredump/gosym.go new file mode 100644 index 00000000..f87e76c7 --- /dev/null +++ b/tools/coredump/gosym.go @@ -0,0 +1,206 @@ +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + + "github.com/peterbourgon/ff/v3/ffcli" + "go.opentelemetry.io/ebpf-profiler/libpf" + "go.opentelemetry.io/ebpf-profiler/reporter" + "go.opentelemetry.io/ebpf-profiler/tools/coredump/modulestore" +) + +type gosymCmd struct { + store *modulestore.Store + casePath string +} + +func newGosymCmd(store *modulestore.Store) *ffcli.Command { + args := &gosymCmd{store: store} + + set := flag.NewFlagSet("gosym", flag.ExitOnError) + set.StringVar(&args.casePath, "case", "", "Path of the test case to debug") + + return &ffcli.Command{ + Name: "gosym", + Exec: args.exec, + ShortUsage: "gosym", + ShortHelp: "Symbolize go test case", + FlagSet: set, + } +} + +func (cmd *gosymCmd) exec(context.Context, []string) (err error) { + // Validate arguments. + if cmd.casePath == "" { + return errors.New("please specify `-case`") + } + + var test *CoredumpTestCase + test, err = readTestCase(cmd.casePath) + if err != nil { + return fmt.Errorf("failed to read test case: %w", err) + } + + if got := len(test.Modules); got != 1 { + return fmt.Errorf("got=%d module but only 1 module is supported right now", got) + } + + binary, err := extractModuleToTempFile(cmd.store, test.Modules[0]) + if err != nil { + return fmt.Errorf("failed to extract binary: %w", err) + } + defer os.Remove(binary) + + addrs := map[libpf.AddressOrLineno]struct{}{} + frames := map[libpf.AddressOrLineno][]*string{} + for _, thread := range test.Threads { + for i, frame := range thread.Frames { + _, addr, err := parseUnsymbolizedFrame(frame) + if err != nil { + continue + } + addrs[addr] = struct{}{} + frames[addr] = append(frames[addr], &thread.Frames[i]) + } + } + + locs, err := goSymbolize(binary, addrs) + if err != nil { + return fmt.Errorf("failed to symbolize: %w", err) + } + + for addr, frame := range locs { + for _, frameS := range frames[addr] { + *frameS = formatSymbolizedFrame(frame, false) + " (" + *frameS + ")" + } + } + + return writeTestCaseJSON(os.Stdout, test) +} + +func extractModuleToTempFile(store *modulestore.Store, m ModuleInfo) (string, error) { + file, err := os.CreateTemp("", "") + if err != nil { + return "", err + } + return file.Name(), store.UnpackModuleToPath(m.Ref, file.Name()) +} + +func goSymbolize(binary string, addrs map[libpf.AddressOrLineno]struct{}) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) { + // Launch addr2line process. + addr2line := exec.Command("go", "tool", "addr2line", binary) + inR, inW := io.Pipe() + outR, outW := io.Pipe() + addr2line.Stdin = inR + addr2line.Stdout = outW + addr2line.Stderr = outW + if err := addr2line.Start(); err != nil { + return nil, err + } + + // Transform addrs into a list. This allows us to figure out which addr2line + // output corresponds to which address. + addrList := make([]libpf.AddressOrLineno, 0, len(addrs)) + for pc, _ := range addrs { + addrList = append(addrList, pc) + } + + // Parse addr2line output and map it to addrs we were given. + frames := map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs{} + scanCh := make(chan error) + go func() { + // Drain the output pipe in case we hit a parsing error. + defer io.Copy(io.Discard, outR) + + var err error + readFrame := addr2LineFrameReader(outR) + for { + var frame *reporter.FrameMetadataArgs + frame, err = readFrame() + if err != nil { + break + } + addr := addrList[len(frames)] + frames[addr] = frame + } + scanCh <- err + }() + + // Write addrList to addr2line stdin. + var writeErr error + writeAddr := addr2LineAddrWriter(inW) + for _, addr := range addrList { + if writeErr = writeAddr(addr); writeErr != nil { + break + } + } + + // Close the input pipe to signal addr2line that we're done. + if err := inW.Close(); err != nil { + return nil, err + // Wait for addr2line to finish. + } else if err := addr2line.Wait(); err != nil { + return nil, err + // Signal the output reader that we're done. + } else if err := outW.Close(); err != nil { + return nil, err + // Wait for the output reader to finish. + } else if err := <-scanCh; err != nil && err != io.EOF { + return nil, err + } + return frames, writeErr +} + +func addr2LineAddrWriter(w io.Writer) func(libpf.AddressOrLineno) error { + return func(addr libpf.AddressOrLineno) error { + _, err := fmt.Fprintf(w, "%x\n", addr) + return err + } +} + +func addr2LineFrameReader(r io.Reader) func() (*reporter.FrameMetadataArgs, error) { + scanner := bufio.NewScanner(r) + scanErr := func() error { + if err := scanner.Err(); err != nil { + return err + } + return io.EOF + } + var pair [2]string + return func() (*reporter.FrameMetadataArgs, error) { + if !scanner.Scan() { + return nil, scanErr() + } + pair[0] = scanner.Text() + if !scanner.Scan() { + return nil, fmt.Errorf("expected second line, but got: %w", scanErr()) + } + pair[1] = scanner.Text() + return linePairToFrame(pair) + } +} + +func linePairToFrame(pair [2]string) (*reporter.FrameMetadataArgs, error) { + var frame reporter.FrameMetadataArgs + frame.FunctionName = pair[0] + file, line, found := strings.Cut(pair[1], ":") + if !found { + return nil, fmt.Errorf("expected file:line but got: %q", pair[1]) + } + lineNum, err := strconv.Atoi(line) + if err != nil { + return nil, fmt.Errorf("invalid line number: %q", line) + } + frame.SourceFile = file + frame.SourceLine = libpf.SourceLineno(lineNum) + return &frame, nil +} diff --git a/tools/coredump/json.go b/tools/coredump/json.go index 388691de..4f904387 100644 --- a/tools/coredump/json.go +++ b/tools/coredump/json.go @@ -8,6 +8,7 @@ package main import ( "encoding/json" "fmt" + "io" "os" "path/filepath" "runtime" @@ -73,13 +74,17 @@ func writeTestCase(path string, c *CoredumpTestCase, allowOverwrite bool) error return fmt.Errorf("failed to create JSON file: %w", err) } - enc := json.NewEncoder(jsonFile) + return writeTestCaseJSON(jsonFile, c) +} + +// writeTestCaseJSON writes a test case to the given writer as JSON. +func writeTestCaseJSON(w io.Writer, c *CoredumpTestCase) error { + enc := json.NewEncoder(w) enc.SetIndent("", " ") enc.SetEscapeHTML(false) if err := enc.Encode(c); err != nil { return fmt.Errorf("JSON Marshall failed: %w", err) } - return nil } diff --git a/tools/coredump/main.go b/tools/coredump/main.go index d24aeb37..9be2501a 100644 --- a/tools/coredump/main.go +++ b/tools/coredump/main.go @@ -58,6 +58,7 @@ func main() { newRebaseCmd(store), newUploadCmd(store), newGdbCmd(store), + newGosymCmd(store), }, Exec: func(context.Context, []string) error { return flag.ErrHelp