Skip to content

Commit

Permalink
coredump: Add gosym sub command
Browse files Browse the repository at this point in the history
gosym symbolizes go test cases using addr2line. This is useful for
debugging the correctness of go stack unwinding results.
  • Loading branch information
felixge committed Nov 10, 2024
1 parent 2ad91df commit 0177cef
Show file tree
Hide file tree
Showing 4 changed files with 248 additions and 6 deletions.
38 changes: 34 additions & 4 deletions tools/coredump/coredump.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"fmt"
"os"
"runtime"
"strconv"
"strings"
"time"
"unsafe"

Expand Down Expand Up @@ -114,16 +116,44 @@ func (c *symbolizationCache) symbolize(ty libpf.FrameType, fileID libpf.FileID,
}

if data, ok := c.symbols[libpf.NewFrameID(fileID, lineNumber)]; ok {
return fmt.Sprintf("%s+%d in %s:%d",
data.FunctionName, data.FunctionOffset,
data.SourceFile, data.SourceLine), nil
return formatSymbolizedFrame(data, true), nil
}

sourceFile, ok := c.files[fileID]
if !ok {
sourceFile = fmt.Sprintf("%08x", fileID)
}
return fmt.Sprintf("%s+0x%x", sourceFile, lineNumber), nil
return formatUnsymbolizedFrame(sourceFile, lineNumber), nil
}

func formatSymbolizedFrame(frame *reporter.FrameMetadataArgs, functionOffsets bool) string {
var funcOffset string
if functionOffsets {
funcOffset = "+" + strconv.Itoa(int(frame.FunctionOffset))
}
return fmt.Sprintf("%s%s in %s:%d",
frame.FunctionName, funcOffset,
frame.SourceFile, frame.SourceLine)
}

func formatUnsymbolizedFrame(file string, addr libpf.AddressOrLineno) string {
return fmt.Sprintf("%s+0x%x", file, addr)
}

func parseUnsymbolizedFrame(frame string) (file string, addr libpf.AddressOrLineno, err error) {
fileS, addrS, found := strings.Cut(frame, "+0x")
if !found {
err = fmt.Errorf("bad frame string: %q", frame)
return
}
file = fileS
var addrU uint64
addrU, err = strconv.ParseUint(addrS, 16, 64)
if err != nil {
return
}
addr = libpf.AddressOrLineno(addrU)
return
}

func ExtractTraces(ctx context.Context, pr process.Process, debug bool,
Expand Down
206 changes: 206 additions & 0 deletions tools/coredump/gosym.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
package main

import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"io"
"os"
"os/exec"
"strconv"
"strings"

"github.com/peterbourgon/ff/v3/ffcli"
"go.opentelemetry.io/ebpf-profiler/libpf"
"go.opentelemetry.io/ebpf-profiler/reporter"
"go.opentelemetry.io/ebpf-profiler/tools/coredump/modulestore"
)

type gosymCmd struct {
store *modulestore.Store
casePath string
}

func newGosymCmd(store *modulestore.Store) *ffcli.Command {
args := &gosymCmd{store: store}

set := flag.NewFlagSet("gosym", flag.ExitOnError)
set.StringVar(&args.casePath, "case", "", "Path of the test case to debug")

return &ffcli.Command{
Name: "gosym",
Exec: args.exec,
ShortUsage: "gosym",
ShortHelp: "Symbolize go test case",
FlagSet: set,
}
}

func (cmd *gosymCmd) exec(context.Context, []string) (err error) {
// Validate arguments.
if cmd.casePath == "" {
return errors.New("please specify `-case`")
}

var test *CoredumpTestCase
test, err = readTestCase(cmd.casePath)
if err != nil {
return fmt.Errorf("failed to read test case: %w", err)
}

if got := len(test.Modules); got != 1 {
return fmt.Errorf("got=%d module but only 1 module is supported right now", got)
}

binary, err := extractModuleToTempFile(cmd.store, test.Modules[0])
if err != nil {
return fmt.Errorf("failed to extract binary: %w", err)
}
defer os.Remove(binary)

addrs := map[libpf.AddressOrLineno]struct{}{}
frames := map[libpf.AddressOrLineno][]*string{}
for _, thread := range test.Threads {
for i, frame := range thread.Frames {
_, addr, err := parseUnsymbolizedFrame(frame)

Check failure on line 67 in tools/coredump/gosym.go

View workflow job for this annotation

GitHub Actions / Lint (amd64)

shadow: declaration of "err" shadows declaration at line 41 (govet)
if err != nil {
continue
}
addrs[addr] = struct{}{}
frames[addr] = append(frames[addr], &thread.Frames[i])
}
}

locs, err := goSymbolize(binary, addrs)
if err != nil {
return fmt.Errorf("failed to symbolize: %w", err)
}

for addr, frame := range locs {
for _, frameS := range frames[addr] {
*frameS = formatSymbolizedFrame(frame, false) + " (" + *frameS + ")"
}
}

return writeTestCaseJSON(os.Stdout, test)
}

func extractModuleToTempFile(store *modulestore.Store, m ModuleInfo) (string, error) {
file, err := os.CreateTemp("", "")
if err != nil {
return "", err
}
return file.Name(), store.UnpackModuleToPath(m.Ref, file.Name())
}

func goSymbolize(binary string, addrs map[libpf.AddressOrLineno]struct{}) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) {
// Launch addr2line process.
addr2line := exec.Command("go", "tool", "addr2line", binary)
inR, inW := io.Pipe()
outR, outW := io.Pipe()
addr2line.Stdin = inR
addr2line.Stdout = outW
addr2line.Stderr = outW
if err := addr2line.Start(); err != nil {
return nil, err
}

// Transform addrs into a list. This allows us to figure out which addr2line
// output corresponds to which address.
addrList := make([]libpf.AddressOrLineno, 0, len(addrs))
for pc, _ := range addrs {
addrList = append(addrList, pc)
}

// Parse addr2line output and map it to addrs we were given.
frames := map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs{}
scanCh := make(chan error)
go func() {
// Drain the output pipe in case we hit a parsing error.
defer io.Copy(io.Discard, outR)

Check failure on line 122 in tools/coredump/gosym.go

View workflow job for this annotation

GitHub Actions / Lint (amd64)

Error return value of `io.Copy` is not checked (errcheck)

var err error
readFrame := addr2LineFrameReader(outR)
for {
var frame *reporter.FrameMetadataArgs
frame, err = readFrame()
if err != nil {
break
}
addr := addrList[len(frames)]
frames[addr] = frame
}
scanCh <- err
}()

// Write addrList to addr2line stdin.
var writeErr error
writeAddr := addr2LineAddrWriter(inW)
for _, addr := range addrList {
if writeErr = writeAddr(addr); writeErr != nil {
break
}
}

// Close the input pipe to signal addr2line that we're done.
if err := inW.Close(); err != nil {
return nil, err
// Wait for addr2line to finish.
} else if err := addr2line.Wait(); err != nil {
return nil, err
// Signal the output reader that we're done.
} else if err := outW.Close(); err != nil {
return nil, err
// Wait for the output reader to finish.
} else if err := <-scanCh; err != nil && err != io.EOF {
return nil, err
}
return frames, writeErr
}

func addr2LineAddrWriter(w io.Writer) func(libpf.AddressOrLineno) error {
return func(addr libpf.AddressOrLineno) error {
_, err := fmt.Fprintf(w, "%x\n", addr)
return err
}
}

func addr2LineFrameReader(r io.Reader) func() (*reporter.FrameMetadataArgs, error) {
scanner := bufio.NewScanner(r)
scanErr := func() error {
if err := scanner.Err(); err != nil {
return err
}
return io.EOF
}
var pair [2]string
return func() (*reporter.FrameMetadataArgs, error) {
if !scanner.Scan() {
return nil, scanErr()
}
pair[0] = scanner.Text()
if !scanner.Scan() {
return nil, fmt.Errorf("expected second line, but got: %w", scanErr())
}
pair[1] = scanner.Text()
return linePairToFrame(pair)
}
}

func linePairToFrame(pair [2]string) (*reporter.FrameMetadataArgs, error) {
var frame reporter.FrameMetadataArgs
frame.FunctionName = pair[0]
file, line, found := strings.Cut(pair[1], ":")
if !found {
return nil, fmt.Errorf("expected file:line but got: %q", pair[1])
}
lineNum, err := strconv.Atoi(line)
if err != nil {
return nil, fmt.Errorf("invalid line number: %q", line)
}
frame.SourceFile = file
frame.SourceLine = libpf.SourceLineno(lineNum)
return &frame, nil
}
9 changes: 7 additions & 2 deletions tools/coredump/json.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ package main
import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
Expand Down Expand Up @@ -73,13 +74,17 @@ func writeTestCase(path string, c *CoredumpTestCase, allowOverwrite bool) error
return fmt.Errorf("failed to create JSON file: %w", err)
}

enc := json.NewEncoder(jsonFile)
return writeTestCaseJSON(jsonFile, c)
}

// writeTestCaseJSON writes a test case to the given writer as JSON.
func writeTestCaseJSON(w io.Writer, c *CoredumpTestCase) error {
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.SetEscapeHTML(false)
if err := enc.Encode(c); err != nil {
return fmt.Errorf("JSON Marshall failed: %w", err)
}

return nil
}

Expand Down
1 change: 1 addition & 0 deletions tools/coredump/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ func main() {
newRebaseCmd(store),
newUploadCmd(store),
newGdbCmd(store),
newGosymCmd(store),
},
Exec: func(context.Context, []string) error {
return flag.ErrHelp
Expand Down

0 comments on commit 0177cef

Please sign in to comment.