coredump: Add gosym sub command

README.md

@@ -476,7 +476,7 @@ The host agent code is tested with three test suites:
   tests. This works great for the user-land portion of the agent, but is unable
   to test any of the unwinding logic and BPF interaction.
 - **coredump test suite**\
-  The coredump test suite (`utils/coredump`) we compile the whole BPF unwinder
+  The coredump test suite (`tools/coredump`) we compile the whole BPF unwinder
   code into a user-mode executable, then use the information from a coredump to
   simulate a realistic environment to test the unwinder code in. The coredump
   suite essentially implements all required BPF helper functions in user-space,

tools/coredump/coredump.go

@@ -10,6 +10,8 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strconv"
+	"strings"
 	"time"
 	"unsafe"
 
@@ -114,16 +116,44 @@ func (c *symbolizationCache) symbolize(ty libpf.FrameType, fileID libpf.FileID,
 	}
 
 	if data, ok := c.symbols[libpf.NewFrameID(fileID, lineNumber)]; ok {
-		return fmt.Sprintf("%s+%d in %s:%d",
-			data.FunctionName, data.FunctionOffset,
-			data.SourceFile, data.SourceLine), nil
+		return formatSymbolizedFrame(data, true), nil
 	}
 
 	sourceFile, ok := c.files[fileID]
 	if !ok {
 		sourceFile = fmt.Sprintf("%08x", fileID)
 	}
-	return fmt.Sprintf("%s+0x%x", sourceFile, lineNumber), nil
+	return formatUnsymbolizedFrame(sourceFile, lineNumber), nil
+}
+
+func formatSymbolizedFrame(frame *reporter.FrameMetadataArgs, functionOffsets bool) string {
+	var funcOffset string
+	if functionOffsets {
+		funcOffset = "+" + strconv.Itoa(int(frame.FunctionOffset))
+	}
+	return fmt.Sprintf("%s%s in %s:%d",
+		frame.FunctionName, funcOffset,
+		frame.SourceFile, frame.SourceLine)
+}
+
+func formatUnsymbolizedFrame(file string, addr libpf.AddressOrLineno) string {
+	return fmt.Sprintf("%s+0x%x", file, addr)
+}
+
+func parseUnsymbolizedFrame(frame string) (file string, addr libpf.AddressOrLineno, err error) {
+	fileS, addrS, found := strings.Cut(frame, "+0x")
+	if !found {
+		err = fmt.Errorf("bad frame string: %q", frame)
+		return
+	}
+	file = fileS
+	var addrU uint64
+	addrU, err = strconv.ParseUint(addrS, 16, 64)
+	if err != nil {
+		return
+	}
+	addr = libpf.AddressOrLineno(addrU)
+	return
 }
 
 func ExtractTraces(ctx context.Context, pr process.Process, debug bool,

tools/coredump/gosym.go

@@ -0,0 +1,206 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"github.com/peterbourgon/ff/v3/ffcli"
+	"go.opentelemetry.io/ebpf-profiler/libpf"
+	"go.opentelemetry.io/ebpf-profiler/reporter"
+	"go.opentelemetry.io/ebpf-profiler/tools/coredump/modulestore"
+)
+
+type gosymCmd struct {
+	store    *modulestore.Store
+	casePath string
+}
+
+func newGosymCmd(store *modulestore.Store) *ffcli.Command {
+	args := &gosymCmd{store: store}
+
+	set := flag.NewFlagSet("gosym", flag.ExitOnError)
+	set.StringVar(&args.casePath, "case", "", "Path of the test case to debug")
+
+	return &ffcli.Command{
+		Name:       "gosym",
+		Exec:       args.exec,
+		ShortUsage: "gosym",
+		ShortHelp:  "Symbolize go test case",
+		FlagSet:    set,
+	}
+}
+
+func (cmd *gosymCmd) exec(context.Context, []string) (err error) {
+	// Validate arguments.
+	if cmd.casePath == "" {
+		return errors.New("please specify `-case`")
+	}
+
+	var test *CoredumpTestCase
+	test, err = readTestCase(cmd.casePath)
+	if err != nil {
+		return fmt.Errorf("failed to read test case: %w", err)
+	}
+
+	if got := len(test.Modules); got != 1 {
+		return fmt.Errorf("got=%d module but only 1 module is supported right now", got)
+	}
+
+	binary, err := extractModuleToTempFile(cmd.store, test.Modules[0])
+	if err != nil {
+		return fmt.Errorf("failed to extract binary: %w", err)
+	}
+	defer os.Remove(binary)
+
+	addrs := map[libpf.AddressOrLineno]struct{}{}
+	frames := map[libpf.AddressOrLineno][]*string{}
+	for _, thread := range test.Threads {
+		for i, frame := range thread.Frames {
+			_, addr, err := parseUnsymbolizedFrame(frame)
+			if err != nil {
+				continue
+			}
+			addrs[addr] = struct{}{}
+			frames[addr] = append(frames[addr], &thread.Frames[i])
+		}
+	}
+
+	locs, err := goSymbolize(binary, addrs)
+	if err != nil {
+		return fmt.Errorf("failed to symbolize: %w", err)
+	}
+
+	for addr, frame := range locs {
+		for _, frameS := range frames[addr] {
+			*frameS = formatSymbolizedFrame(frame, false) + " (" + *frameS + ")"
+		}
+	}
+
+	return writeTestCaseJSON(os.Stdout, test)
+}
+
+func extractModuleToTempFile(store *modulestore.Store, m ModuleInfo) (string, error) {
+	file, err := os.CreateTemp("", "")
+	if err != nil {
+		return "", err
+	}
+	return file.Name(), store.UnpackModuleToPath(m.Ref, file.Name())
+}
+
+func goSymbolize(binary string, addrs map[libpf.AddressOrLineno]struct{}) (map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs, error) {
+	// Launch addr2line process.
+	addr2line := exec.Command("go", "tool", "addr2line", binary)
+	inR, inW := io.Pipe()
+	outR, outW := io.Pipe()
+	addr2line.Stdin = inR
+	addr2line.Stdout = outW
+	addr2line.Stderr = outW
+	if err := addr2line.Start(); err != nil {
+		return nil, err
+	}
+
+	// Transform addrs into a list. This allows us to figure out which addr2line
+	// output corresponds to which address.
+	addrList := make([]libpf.AddressOrLineno, 0, len(addrs))
+	for pc, _ := range addrs {
+		addrList = append(addrList, pc)
+	}
+
+	// Parse addr2line output and map it to addrs we were given.
+	frames := map[libpf.AddressOrLineno]*reporter.FrameMetadataArgs{}
+	scanCh := make(chan error)
+	go func() {
+		// Drain the output pipe in case we hit a parsing error.
+		defer io.Copy(io.Discard, outR)
+
+		var err error
+		readFrame := addr2LineFrameReader(outR)
+		for {
+			var frame *reporter.FrameMetadataArgs
+			frame, err = readFrame()
+			if err != nil {
+				break
+			}
+			addr := addrList[len(frames)]
+			frames[addr] = frame
+		}
+		scanCh <- err
+	}()
+
+	// Write addrList to addr2line stdin.
+	var writeErr error
+	writeAddr := addr2LineAddrWriter(inW)
+	for _, addr := range addrList {
+		if writeErr = writeAddr(addr); writeErr != nil {
+			break
+		}
+	}
+
+	// Close the input pipe to signal addr2line that we're done.
+	if err := inW.Close(); err != nil {
+		return nil, err
+		// Wait for addr2line to finish.
+	} else if err := addr2line.Wait(); err != nil {
+		return nil, err
+		// Signal the output reader that we're done.
+	} else if err := outW.Close(); err != nil {
+		return nil, err
+		// Wait for the output reader to finish.
+	} else if err := <-scanCh; err != nil && err != io.EOF {
+		return nil, err
+	}
+	return frames, writeErr
+}
+
+func addr2LineAddrWriter(w io.Writer) func(libpf.AddressOrLineno) error {
+	return func(addr libpf.AddressOrLineno) error {
+		_, err := fmt.Fprintf(w, "%x\n", addr)
+		return err
+	}
+}
+
+func addr2LineFrameReader(r io.Reader) func() (*reporter.FrameMetadataArgs, error) {
+	scanner := bufio.NewScanner(r)
+	scanErr := func() error {
+		if err := scanner.Err(); err != nil {
+			return err
+		}
+		return io.EOF
+	}
+	var pair [2]string
+	return func() (*reporter.FrameMetadataArgs, error) {
+		if !scanner.Scan() {
+			return nil, scanErr()
+		}
+		pair[0] = scanner.Text()
+		if !scanner.Scan() {
+			return nil, fmt.Errorf("expected second line, but got: %w", scanErr())
+		}
+		pair[1] = scanner.Text()
+		return linePairToFrame(pair)
+	}
+}
+
+func linePairToFrame(pair [2]string) (*reporter.FrameMetadataArgs, error) {
+	var frame reporter.FrameMetadataArgs
+	frame.FunctionName = pair[0]
+	file, line, found := strings.Cut(pair[1], ":")
+	if !found {
+		return nil, fmt.Errorf("expected file:line but got: %q", pair[1])
+	}
+	lineNum, err := strconv.Atoi(line)
+	if err != nil {
+		return nil, fmt.Errorf("invalid line number: %q", line)
+	}
+	frame.SourceFile = file
+	frame.SourceLine = libpf.SourceLineno(lineNum)
+	return &frame, nil
+}

tools/coredump/json.go

@@ -8,6 +8,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -73,13 +74,17 @@ func writeTestCase(path string, c *CoredumpTestCase, allowOverwrite bool) error
 		return fmt.Errorf("failed to create JSON file: %w", err)
 	}
 
-	enc := json.NewEncoder(jsonFile)
+	return writeTestCaseJSON(jsonFile, c)
+}
+
+// writeTestCaseJSON writes a test case to the given writer as JSON.
+func writeTestCaseJSON(w io.Writer, c *CoredumpTestCase) error {
+	enc := json.NewEncoder(w)
 	enc.SetIndent("", "  ")
 	enc.SetEscapeHTML(false)
 	if err := enc.Encode(c); err != nil {
 		return fmt.Errorf("JSON Marshall failed: %w", err)
 	}
-
 	return nil
 }
 

tools/coredump/main.go

@@ -58,6 +58,7 @@ func main() {
 			newRebaseCmd(store),
 			newUploadCmd(store),
 			newGdbCmd(store),
+			newGosymCmd(store),
 		},
 		Exec: func(context.Context, []string) error {
 			return flag.ErrHelp