From 507be3fbea5febb2fcd946f3d4d4f153e66a7ff9 Mon Sep 17 00:00:00 2001 From: Holger Hans Peter Freyther Date: Thu, 14 Oct 2021 23:02:46 +0800 Subject: [PATCH 1/3] perf: Start parsing files created by JITs on Linux Parse the Linux jit-interface perf files[1] and sort by start addresses. There will be some dragons to make this work with containerized workloads (e.g. map the pid to nspid and then access /tmp/perf-${nspid}.map inside the right rootfs and be aware that this is entirely user controlled). [1] https://github.com/torvalds/linux/blob/master/tools/perf/Documentation/jit-interface.txt Signed-off-by: Holger Hans Peter Freyther --- pkg/perf/perf.go | 72 ++++++++++++++++++++++++++++++++++++++++ pkg/perf/perf_test.go | 77 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 pkg/perf/perf.go create mode 100644 pkg/perf/perf_test.go diff --git a/pkg/perf/perf.go b/pkg/perf/perf.go new file mode 100644 index 000000000..a0e962227 --- /dev/null +++ b/pkg/perf/perf.go @@ -0,0 +1,72 @@ +// Copyright 2021 The Parca Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package perf + +import ( + "bufio" + "fmt" + "io/fs" + "sort" + "strconv" + "strings" +) + +type PerfMapAddr struct { + Start uint64 + End uint64 + Symbol string +} + +type PerfMap struct { + addrs []PerfMapAddr +} + +func PerfReadMap(fs fs.FS, fileName string) (PerfMap, error) { + fd, err := fs.Open(fileName) + if err != nil { + return PerfMap{}, err + } + defer fd.Close() + + s := bufio.NewScanner(fd) + addrs := make([]PerfMapAddr, 0) + for s.Scan() { + l := strings.SplitN(s.Text(), " ", 3) + if len(l) < 3 { + return PerfMap{}, fmt.Errorf("splitting failed: %v", l) + + } + + start, err := strconv.ParseUint(l[0], 16, 64) + if err != nil { + return PerfMap{}, fmt.Errorf("parsing start failed on %v: %w", l, err) + } + size, err := strconv.ParseUint(l[1], 16, 64) + if err != nil { + return PerfMap{}, fmt.Errorf("parsing end failed on %v: %w", l, err) + } + if start+size < start { + return PerfMap{}, fmt.Errorf("overflowed mapping: %v", l) + } + addrs = append(addrs, PerfMapAddr{start, start + size, l[2]}) + } + // Sorted by end address to allow binary search during look-up. End to find + // the (closest) address _before_ the end. This could be an inlined instruction + // within a larger blob. + sort.Slice(addrs, func(i, j int) bool { + return addrs[i].End < addrs[j].End + }) + return PerfMap{addrs: addrs}, s.Err() +} diff --git a/pkg/perf/perf_test.go b/pkg/perf/perf_test.go new file mode 100644 index 000000000..407ddb1c6 --- /dev/null +++ b/pkg/perf/perf_test.go @@ -0,0 +1,77 @@ +// Copyright 2021 The Parca Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package perf + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/parca-dev/parca-agent/pkg/testutil" +) + +// See https://github.com/torvalds/linux/blob/master/tools/perf/Documentation/jit-interface.txt +const perfMap = `3ef414c0 398 RegExp:[{(] +3ef418a0 398 RegExp:[})] +59ed4102 26 LazyCompile:~REPLServer.self.writer repl.js:514 +59ed44ea 146 LazyCompile:~inspect internal/util/inspect.js:152 +59ed4e4a 148 LazyCompile:~formatValue internal/util/inspect.js:456 +59ed558a 25f LazyCompile:~formatPrimitive internal/util/inspect.js:768 +59ed5d62 35 LazyCompile:~formatNumber internal/util/inspect.js:761 +59ed5fca 5d LazyCompile:~stylizeWithColor internal/util/inspect.js:267 +4edd2e52 65 LazyCompile:~Domain.exit domain.js:284 +4edd30ea 14b LazyCompile:~lastIndexOf native array.js:618 +4edd3522 35 LazyCompile:~online internal/repl.js:157 +4edd37f2 ec LazyCompile:~setTimeout timers.js:388 +4edd3cca b0 LazyCompile:~Timeout internal/timers.js:55 +4edd40ba 55 LazyCompile:~initAsyncResource internal/timers.js:45 +4edd42da f LazyCompile:~exports.active timers.js:151 +4edd457a cb LazyCompile:~insert timers.js:167 +4edd4962 50 LazyCompile:~TimersList timers.js:195 +4edd4cea 37 LazyCompile:~append internal/linkedlist.js:29 +4edd4f12 35 LazyCompile:~remove internal/linkedlist.js:15 +4edd5132 d LazyCompile:~isEmpty internal/linkedlist.js:44 +4edd529a 21 LazyCompile:~ok assert.js:345 +4edd555a 68 LazyCompile:~innerOk assert.js:317 +4edd59a2 27 LazyCompile:~processTimers timers.js:220 +4edd5d9a 197 LazyCompile:~listOnTimeout timers.js:226 +4edd6352 15 LazyCompile:~peek internal/linkedlist.js:9 +4edd66ca a1 LazyCompile:~tryOnTimeout timers.js:292 +4edd6a02 86 LazyCompile:~ontimeout timers.js:429 +4edd7132 d7 LazyCompile:~process.kill internal/process/per_thread.js:173` + +func TestPerfMapParse(t *testing.T) { + fs := testutil.NewFakeFS(map[string][]byte{ + "/tmp/perf-123.map": []byte(perfMap), + }) + + res, err := PerfReadMap(fs, "/tmp/perf-123.map") + require.NoError(t, err) + require.Len(t, res.addrs, 28) + // Check for 4edd3cca B0 LazyCompile:~Timeout internal/timers.js:55 + require.Equal(t, res.addrs[12], PerfMapAddr{0x4edd4f12, 0x4edd4f47, "LazyCompile:~remove internal/linkedlist.js:15"}) +} + +func BenchmarkPerfMapParse(b *testing.B) { + fs := testutil.NewFakeFS(map[string][]byte{ + "/tmp/perf-123.map": []byte(perfMap), + }) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, err := PerfReadMap(fs, "/tmp/perf-123.map") + require.NoError(b, err) + } +} From e328dda28d66dbab21cfc64796ebd793179ac7c0 Mon Sep 17 00:00:00 2001 From: Holger Hans Peter Freyther Date: Sat, 16 Oct 2021 22:37:54 +0800 Subject: [PATCH 2/3] perf: Implement and test symbol look-up. We aproach this from the end address and then check that the address is after the start address. Signed-off-by: Holger Hans Peter Freyther --- pkg/perf/perf.go | 15 +++++++++++++++ pkg/perf/perf_test.go | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/pkg/perf/perf.go b/pkg/perf/perf.go index a0e962227..065be188d 100644 --- a/pkg/perf/perf.go +++ b/pkg/perf/perf.go @@ -33,6 +33,10 @@ type PerfMap struct { addrs []PerfMapAddr } +var ( + NoSymbolFound = errors.New("no symbol found") +) + func PerfReadMap(fs fs.FS, fileName string) (PerfMap, error) { fd, err := fs.Open(fileName) if err != nil { @@ -70,3 +74,14 @@ func PerfReadMap(fs fs.FS, fileName string) (PerfMap, error) { }) return PerfMap{addrs: addrs}, s.Err() } + +func (p *PerfMap) Lookup(addr uint64) (string, error) { + idx := sort.Search(len(p.addrs), func(i int) bool { + return addr < p.addrs[i].End + }) + if idx == len(p.addrs) || p.addrs[idx].Start > addr { + return "", NoSymbolFound + } + + return p.addrs[idx].Symbol, nil +} diff --git a/pkg/perf/perf_test.go b/pkg/perf/perf_test.go index 407ddb1c6..3f40410a9 100644 --- a/pkg/perf/perf_test.go +++ b/pkg/perf/perf_test.go @@ -62,6 +62,14 @@ func TestPerfMapParse(t *testing.T) { require.Len(t, res.addrs, 28) // Check for 4edd3cca B0 LazyCompile:~Timeout internal/timers.js:55 require.Equal(t, res.addrs[12], PerfMapAddr{0x4edd4f12, 0x4edd4f47, "LazyCompile:~remove internal/linkedlist.js:15"}) + + // Look-up a symbol. + sym, err := res.Lookup(0x4edd4f12 + 4) + require.NoError(t, err) + require.Equal(t, sym, "LazyCompile:~remove internal/linkedlist.js:15") + + _, err = res.Lookup(0xFFFFFFFF) + require.ErrorIs(t, err, NoSymbolFound) } func BenchmarkPerfMapParse(b *testing.B) { From b0302cb544a54be636892d98fcaa3d8f86c880a5 Mon Sep 17 00:00:00 2001 From: Holger Hans Peter Freyther Date: Sat, 16 Oct 2021 23:36:38 +0800 Subject: [PATCH 3/3] perf: Attempt to load the perf map and resolve symbols Signed-off-by: Holger Hans Peter Freyther --- pkg/agent/profile.go | 32 ++++++++++++++++++++++++ pkg/perf/perf.go | 59 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/pkg/agent/profile.go b/pkg/agent/profile.go index 5ee56fb75..5960dec03 100644 --- a/pkg/agent/profile.go +++ b/pkg/agent/profile.go @@ -42,6 +42,7 @@ import ( "github.com/parca-dev/parca-agent/pkg/debuginfo" "github.com/parca-dev/parca-agent/pkg/ksym" "github.com/parca-dev/parca-agent/pkg/maps" + "github.com/parca-dev/parca-agent/pkg/perf" ) //go:embed parca-agent.bpf.o @@ -90,6 +91,8 @@ type CgroupProfiler struct { mtx *sync.RWMutex lastProfileTakenAt time.Time lastError error + + perfCache *perf.PerfCache } func NewCgroupProfiler( @@ -111,6 +114,7 @@ func NewCgroupProfiler( profilingDuration: profilingDuration, sink: sink, pidMappingFileCache: maps.NewPidMappingFileCache(logger), + perfCache: perf.NewPerfCache(logger), writeClient: writeClient, debugInfoExtractor: debuginfo.NewExtractor( log.With(logger, "component", "debuginfoextractor"), @@ -265,6 +269,7 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, now time.Time, counts, File: "[kernel.kallsyms]", } kernelFunctions := map[uint64]*profile.Function{} + userFunctions := map[[2]uint64]*profile.Function{} // 2 uint64 1 for PID and 1 for Addr locations := []*profile.Location{} @@ -370,6 +375,12 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, now time.Time, counts, } // User stack + perfMap, err := p.perfCache.CacheForPid(pid) + if err != nil { + // We expect only a minority of processes to have a JIT and produce + // the perf map. + level.Debug(p.logger).Log("msg", "no perfmap", "err", err) + } for _, addr := range stack[:stackDepth] { if addr != uint64(0) { key := [2]uint64{uint64(pid), addr} @@ -385,6 +396,22 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, now time.Time, counts, Address: addr, Mapping: m, } + + // Does this addr point to JITed code? + if perfMap != nil { + // TODO(zecke): Log errors other than perf.NoSymbolFound + jitFunction, ok := userFunctions[key] + if !ok { + if sym, err := perfMap.Lookup(addr); err == nil { + jitFunction = &profile.Function{Name: sym} + userFunctions[key] = jitFunction + } + } + if jitFunction != nil { + l.Line = []profile.Line{{Function: jitFunction}} + } + } + locations = append(locations, l) locationIndices[key] = locationIndex } @@ -440,6 +467,11 @@ func (p *CgroupProfiler) profileLoop(ctx context.Context, now time.Time, counts, kernelMapping.ID = uint64(len(prof.Mapping)) + 1 prof.Mapping = append(prof.Mapping, kernelMapping) + for _, f := range userFunctions { + f.ID = uint64(len(prof.Function)) + 1 + prof.Function = append(prof.Function, f) + } + p.debugInfoExtractor.EnsureUploaded(ctx, buildIDFiles) buf := bytes.NewBuffer(nil) diff --git a/pkg/perf/perf.go b/pkg/perf/perf.go index 065be188d..4e0dab024 100644 --- a/pkg/perf/perf.go +++ b/pkg/perf/perf.go @@ -16,13 +16,26 @@ package perf import ( "bufio" + "errors" "fmt" "io/fs" + "os" "sort" "strconv" "strings" + + "github.com/go-kit/log" + + "github.com/parca-dev/parca-agent/pkg/hash" ) +type PerfCache struct { + fs fs.FS + logger log.Logger + cache map[uint32]*PerfMap + pidMapHash map[uint32]uint64 +} + type PerfMapAddr struct { Start uint64 End uint64 @@ -33,10 +46,16 @@ type PerfMap struct { addrs []PerfMapAddr } +type realfs struct{} + var ( NoSymbolFound = errors.New("no symbol found") ) +func (f *realfs) Open(name string) (fs.File, error) { + return os.Open(name) +} + func PerfReadMap(fs fs.FS, fileName string) (PerfMap, error) { fd, err := fs.Open(fileName) if err != nil { @@ -85,3 +104,43 @@ func (p *PerfMap) Lookup(addr uint64) (string, error) { return p.addrs[idx].Symbol, nil } + +func NewPerfCache(logger log.Logger) *PerfCache { + return &PerfCache{ + fs: &realfs{}, + logger: logger, + cache: map[uint32]*PerfMap{}, + pidMapHash: map[uint32]uint64{}, + } +} + +// CacheForPid returns the PerfMap for the given pid if it exists. +func (p *PerfCache) CacheForPid(pid uint32) (*PerfMap, error) { + // NOTE(zecke): There are various limitations and things to note. + // 1st) The input file is "tainted" and under control by the user. By all + // means it could be an infinitely large. + // 2nd) There might be a file called /tmp/perf-${nspid}.txt but that might + // be in a different mount_namespace(7) and pid_namespace(7). We don't + // map these yet. Using /proc/$pid/tmp/perf-$pid.txt is not enough and + // hence containerized workloads are broken. + + perfFile := fmt.Sprintf("/tmp/perf-%d.map", pid) + // TODO(zecke): Log other than file not found errors? + h, err := hash.File(p.fs, perfFile) + if err != nil { + return nil, err + } + + if p.pidMapHash[pid] == h { + return p.cache[pid], nil + } + + m, err := PerfReadMap(p.fs, perfFile) + if err != nil { + return nil, err + } + + p.cache[pid] = &m + p.pidMapHash[pid] = h // TODO(zecke): Resolve time of check/time of use. + return &m, nil +}