From f44d74b5ae1bc4dc3f2eb8201deb62cac36f8f63 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 7 Aug 2024 20:20:25 -0400 Subject: [PATCH 1/5] feat(profiling): Annotate flamegraph with profile data This annotates the flamegraph with metadata that allows us to link to a profile regardless if it's a transaction based profile or continous profile. --- internal/chunk/chunk_test.go | 8 ++ internal/chunk/readjob.go | 12 +-- internal/flamegraph/flamegraph.go | 98 ++++++++++++++++++---- internal/flamegraph/flamegraph_test.go | 108 ++++++++++++++++++++++++- internal/metrics/metrics.go | 19 +++-- internal/nodetree/nodetree.go | 13 +-- internal/profile/android_test.go | 2 +- internal/sample/sample_test.go | 11 +++ internal/speedscope/speedscope.go | 5 +- internal/utils/structs.go | 55 ++++++++++--- 10 files changed, 282 insertions(+), 49 deletions(-) diff --git a/internal/chunk/chunk_test.go b/internal/chunk/chunk_test.go index b6be8e5..40bc550 100644 --- a/internal/chunk/chunk_test.go +++ b/internal/chunk/chunk_test.go @@ -7,6 +7,7 @@ import ( "github.com/getsentry/vroom/internal/nodetree" "github.com/getsentry/vroom/internal/platform" "github.com/getsentry/vroom/internal/testutil" + "github.com/getsentry/vroom/internal/utils" ) func TestCallTrees(t *testing.T) { @@ -47,6 +48,7 @@ func TestCallTrees(t *testing.T) { StartNS: 10_000_000, Frame: frame.Frame{Function: "function0"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), Children: []*nodetree.Node{ { DurationNS: 40_000_000, @@ -58,6 +60,7 @@ func TestCallTrees(t *testing.T) { SampleCount: 2, Frame: frame.Frame{Function: "function1"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), Children: []*nodetree.Node{ { DurationNS: 10_000_000, @@ -69,6 +72,7 @@ func TestCallTrees(t *testing.T) { StartNS: 40_000_000, Frame: frame.Frame{Function: "function2"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), }, }, }, @@ -108,6 +112,7 @@ func TestCallTrees(t *testing.T) { StartNS: 10_000_000, Frame: frame.Frame{Function: "function0"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), Children: []*nodetree.Node{ { DurationNS: 30_000_000, @@ -119,6 +124,7 @@ func TestCallTrees(t *testing.T) { StartNS: 10_000_000, Frame: frame.Frame{Function: "function1"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), }, }, }, @@ -158,6 +164,7 @@ func TestCallTrees(t *testing.T) { StartNS: 10_000_000, Frame: frame.Frame{Function: "function0"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), }, { DurationNS: 10_000_000, @@ -169,6 +176,7 @@ func TestCallTrees(t *testing.T) { StartNS: 20_000_000, Frame: frame.Frame{Function: "function1"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), }, }, }, diff --git a/internal/chunk/readjob.go b/internal/chunk/readjob.go index 27f7f29..60f9703 100644 --- a/internal/chunk/readjob.go +++ b/internal/chunk/readjob.go @@ -15,6 +15,7 @@ type ( ProjectID uint64 ProfilerID string ChunkID string + TransactionID string ThreadID *string Start uint64 End uint64 @@ -22,11 +23,12 @@ type ( } ReadJobResult struct { - Err error - Chunk Chunk - ThreadID *string - Start uint64 - End uint64 + Err error + Chunk Chunk + TransactionID string + ThreadID *string + Start uint64 + End uint64 } ) diff --git a/internal/flamegraph/flamegraph.go b/internal/flamegraph/flamegraph.go index 1fb5541..b79dcbb 100644 --- a/internal/flamegraph/flamegraph.go +++ b/internal/flamegraph/flamegraph.go @@ -130,7 +130,7 @@ func GetFlamegraphFromProfiles( for pair := range callTreesQueue { profileID := pair.First for _, callTree := range pair.Second { - addCallTreeToFlamegraph(&flamegraphTree, callTree, profileID) + addCallTreeToFlamegraph(&flamegraphTree, callTree, annotateWithProfileID(profileID)) } countProfAggregated++ } @@ -157,14 +157,26 @@ func sumNodesSampleCount(nodes []*nodetree.Node) int { return c } -func addCallTreeToFlamegraph(flamegraphTree *[]*nodetree.Node, callTree []*nodetree.Node, profileID string) { +func annotateWithProfileID(profileID string) func(n *nodetree.Node) { + return func(n *nodetree.Node) { + n.ProfileIDs[profileID] = void + } +} + +func annotateWithProfileExample(example utils.ExampleMetadata) func(n *nodetree.Node) { + return func(n *nodetree.Node) { + n.Profiles[example] = void + } +} + +func addCallTreeToFlamegraph(flamegraphTree *[]*nodetree.Node, callTree []*nodetree.Node, annotate func(n *nodetree.Node)) { for _, node := range callTree { if existingNode := getMatchingNode(flamegraphTree, node); existingNode != nil { existingNode.SampleCount += node.SampleCount existingNode.DurationNS += node.DurationNS - addCallTreeToFlamegraph(&existingNode.Children, node.Children, profileID) + addCallTreeToFlamegraph(&existingNode.Children, node.Children, annotate) if node.SampleCount > sumNodesSampleCount(node.Children) { - existingNode.ProfileIDs[profileID] = void + annotate(existingNode) } } else { *flamegraphTree = append(*flamegraphTree, node) @@ -173,27 +185,27 @@ func addCallTreeToFlamegraph(flamegraphTree *[]*nodetree.Node, callTree []*nodet // to the right children along the branch yet, // therefore we call a utility that walk the branch // and does it - expandCallTreeWithProfileID(node, profileID) + expandCallTreeWithProfileID(node, annotate) } } } -func expandCallTreeWithProfileID(node *nodetree.Node, profileID string) { +func expandCallTreeWithProfileID(node *nodetree.Node, annotate func(n *nodetree.Node)) { // leaf frames: we must add the profileID if node.Children == nil { - node.ProfileIDs[profileID] = void + annotate(node) } else { childrenSampleCount := 0 for _, child := range node.Children { childrenSampleCount += child.SampleCount - expandCallTreeWithProfileID(child, profileID) + expandCallTreeWithProfileID(child, annotate) } // If the children's sample count is less than the current // nodes sample count, it means there are some samples // ending at the current node. In this case, this node // should also contain the profile ID if node.SampleCount > childrenSampleCount { - node.ProfileIDs[profileID] = void + annotate(node) } } } @@ -201,12 +213,15 @@ func expandCallTreeWithProfileID(node *nodetree.Node, profileID string) { type flamegraph struct { samples [][]int samplesProfileIDs [][]int + samplesProfiles [][]int sampleCounts []uint64 sampleDurationsNs []uint64 frames []speedscope.Frame framesIndex map[string]int profilesIDsIndex map[string]int profilesIDs []string + profilesIndex map[utils.ExampleMetadata]int + profiles []utils.ExampleMetadata endValue uint64 minFreq int } @@ -217,6 +232,7 @@ func toSpeedscope(trees []*nodetree.Node, minFreq int, projectID uint64) speedsc framesIndex: make(map[string]int), minFreq: minFreq, profilesIDsIndex: make(map[string]int), + profilesIndex: make(map[utils.ExampleMetadata]int), samples: make([][]int, 0), sampleCounts: make([]uint64, 0), } @@ -247,6 +263,7 @@ func toSpeedscope(trees []*nodetree.Node, minFreq int, projectID uint64) speedsc Shared: speedscope.SharedData{ Frames: fd.frames, ProfileIDs: fd.profilesIDs, + Profiles: fd.profiles, }, Profiles: aggProfiles, } @@ -284,7 +301,13 @@ func (f *flamegraph) visitCalltree(node *nodetree.Node, currentStack *[]int) { // base case (when we reach leaf frames) if node.Children == nil { - f.addSample(currentStack, uint64(node.SampleCount), node.DurationNS, node.ProfileIDs) + f.addSample( + currentStack, + uint64(node.SampleCount), + node.DurationNS, + node.ProfileIDs, + node.Profiles, + ) } else { totChildrenSampleCount := 0 var totChildrenDuration uint64 @@ -301,20 +324,33 @@ func (f *flamegraph) visitCalltree(node *nodetree.Node, currentStack *[]int) { diffCount := node.SampleCount - totChildrenSampleCount diffDuration := node.DurationNS - totChildrenDuration if diffCount >= f.minFreq { - f.addSample(currentStack, uint64(diffCount), diffDuration, node.ProfileIDs) + f.addSample( + currentStack, + uint64(diffCount), + diffDuration, + node.ProfileIDs, + node.Profiles, + ) } } // pop last element before returning *currentStack = (*currentStack)[:len(*currentStack)-1] } -func (f *flamegraph) addSample(stack *[]int, count uint64, duration uint64, profileIDs map[string]struct{}) { +func (f *flamegraph) addSample( + stack *[]int, + count uint64, + duration uint64, + profileIDs map[string]struct{}, + profiles map[utils.ExampleMetadata]struct{}, +) { cp := make([]int, len(*stack)) copy(cp, *stack) f.samples = append(f.samples, cp) f.sampleCounts = append(f.sampleCounts, count) f.sampleDurationsNs = append(f.sampleDurationsNs, duration) f.samplesProfileIDs = append(f.samplesProfileIDs, f.getProfileIDsIndices(profileIDs)) + f.samplesProfiles = append(f.samplesProfiles, f.getProfilesIndices(profiles)) f.endValue += count } @@ -332,6 +368,20 @@ func (f *flamegraph) getProfileIDsIndices(profileIDs map[string]struct{}) []int return indices } +func (f *flamegraph) getProfilesIndices(profiles map[utils.ExampleMetadata]struct{}) []int { + indices := make([]int, 0, len(profiles)) + for i := range profiles { + if idx, ok := f.profilesIndex[i]; ok { + indices = append(indices, idx) + } else { + indices = append(indices, len(f.profiles)) + f.profilesIndex[i] = len(f.profiles) + f.profiles = append(f.profiles, i) + } + } + return indices +} + func GetFlamegraphFromChunks( ctx context.Context, organizationID uint64, @@ -390,7 +440,7 @@ func GetFlamegraphFromChunks( intervals := []utils.Interval{interval} for _, callTree := range callTrees { slicedTree := sliceCallTree(&callTree, &intervals) - addCallTreeToFlamegraph(&flamegraphTree, slicedTree, result.Chunk.ID) + addCallTreeToFlamegraph(&flamegraphTree, slicedTree, annotateWithProfileID(result.Chunk.ID)) } } countChunksAggregated++ @@ -472,9 +522,12 @@ func GetFlamegraphFromCandidates( continue } + annotate := annotateWithProfileExample( + utils.NewExampleFromProfileID(result.Profile.ProjectID(), result.Profile.ID()), + ) + for _, callTree := range profileCallTrees { - // TODO: properly pass the profile ID around - addCallTreeToFlamegraph(&flamegraphTree, callTree, "") + addCallTreeToFlamegraph(&flamegraphTree, callTree, annotate) } // if metrics aggregator is not null, while we're at it, // compute the metrics as well @@ -492,6 +545,18 @@ func GetFlamegraphFromCandidates( continue } + annotate := annotateWithProfileExample( + utils.NewExampleFromProfilerChunk( + result.Chunk.ProjectID, + result.Chunk.ProfilerID, + result.Chunk.ID, + result.TransactionID, + result.ThreadID, + result.Start, + result.End, + ), + ) + for _, callTree := range chunkCallTrees { if result.Start > 0 && result.End > 0 { interval := utils.Interval{ @@ -500,8 +565,7 @@ func GetFlamegraphFromCandidates( } callTree = sliceCallTree(&callTree, &[]utils.Interval{interval}) } - // TODO: properly pass the profile ID around - addCallTreeToFlamegraph(&flamegraphTree, callTree, "") + addCallTreeToFlamegraph(&flamegraphTree, callTree, annotate) } // if metrics aggregator is not null, while we're at it, // compute the metrics as well diff --git a/internal/flamegraph/flamegraph_test.go b/internal/flamegraph/flamegraph_test.go index 7bb2f62..7ce6388 100644 --- a/internal/flamegraph/flamegraph_test.go +++ b/internal/flamegraph/flamegraph_test.go @@ -14,6 +14,7 @@ import ( "github.com/getsentry/vroom/internal/speedscope" "github.com/getsentry/vroom/internal/testutil" "github.com/getsentry/vroom/internal/timeutil" + "github.com/getsentry/vroom/internal/utils" ) func TestFlamegraphAggregation(t *testing.T) { @@ -194,7 +195,7 @@ func TestFlamegraphAggregation(t *testing.T) { if err != nil { t.Fatalf("error when generating calltrees: %v", err) } - addCallTreeToFlamegraph(&ft, callTrees[0], p.ID()) + addCallTreeToFlamegraph(&ft, callTrees[0], annotateWithProfileID(p.ID())) } if diff := testutil.Diff(toSpeedscope(ft, 1, 99), test.output, options); diff != "" { @@ -203,3 +204,108 @@ func TestFlamegraphAggregation(t *testing.T) { }) } } + +func TestAnnotatingWithExamples(t *testing.T) { + threadID := "0" + + tests := []struct { + name string + callTrees []*nodetree.Node + examples []utils.ExampleMetadata + output speedscope.Output + }{ + { + name: "Annotate with profile id", + callTrees: []*nodetree.Node{ + { + DurationNS: 40_000_000, + EndNS: 50_000_000, + StartNS: 10_000_000, + Fingerprint: 14164357600995800812, + IsApplication: true, + Name: "function1", + SampleCount: 2, + Frame: frame.Frame{Function: "function1"}, + ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), + Children: []*nodetree.Node{ + { + DurationNS: 10_000_000, + EndNS: 50_000_000, + Fingerprint: 9531802423075301657, + IsApplication: true, + Name: "function2", + SampleCount: 1, + StartNS: 40_000_000, + Frame: frame.Frame{Function: "function2"}, + ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), + }, + }, + }, + }, + examples: []utils.ExampleMetadata{ + utils.NewExampleFromProfileID(1, "2"), + utils.NewExampleFromProfilerChunk(3, "4", "5", "6", &threadID, 10_000_000, 50_000_000), + }, + output: speedscope.Output{ + Metadata: speedscope.ProfileMetadata{ + ProfileView: speedscope.ProfileView{ + ProjectID: 99, + }, + }, + Profiles: []interface{}{ + speedscope.SampledProfile{ + EndValue: 4, + IsMainThread: true, + Samples: [][]int{ + {0, 1}, + {0}, + }, + SamplesProfiles: [][]int{{}, {}}, + Type: "sampled", + Unit: "count", + Weights: []uint64{2, 2}, + SampleCounts: []uint64{2, 2}, + SampleDurationsNs: []uint64{20_000_000, 60_000_000}, + }, + }, + Shared: speedscope.SharedData{ + Frames: []speedscope.Frame{ + {Name: "function1", IsApplication: true}, + {Name: "function2", IsApplication: true}, + }, + Profiles: []utils.ExampleMetadata{ + { + ProjectID: 1, + ProfileID: "2", + }, + { + ProjectID: 3, + ProfilerID: "4", + ChunkID: "5", + TransactionID: "6", + ThreadID: &threadID, + Start: 0.01, + End: 0.05, + }, + }, + }, + }, + }, + } + + options := cmp.Options{} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var ft []*nodetree.Node + for _, example := range test.examples { + addCallTreeToFlamegraph(&ft, test.callTrees, annotateWithProfileExample(example)) + } + if diff := testutil.Diff(toSpeedscope(ft, 1, 99), test.output, options); diff != "" { + t.Fatalf("Result mismatch: got - want +\n%s", diff) + } + }) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 87a1ff2..4efc796 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -193,6 +193,7 @@ func (ma *Aggregator) GetMetricsFromCandidates( ProjectID: candidate.ProjectID, ProfilerID: candidate.ProfilerID, ChunkID: candidate.ChunkID, + TransactionID: candidate.TransactionID, ThreadID: candidate.ThreadID, Start: candidate.Start, End: candidate.End, @@ -225,9 +226,7 @@ func (ma *Aggregator) GetMetricsFromCandidates( hub.CaptureException(err) continue } - resultMetadata = utils.ExampleMetadata{ - ProfileID: resultMetadata.ProfileID, - } + resultMetadata = utils.NewExampleFromProfileID(result.Profile.ProjectID(), result.Profile.ID()) functions := CapAndFilterFunctions(ExtractFunctionsFromCallTrees(profileCallTrees), int(ma.MaxUniqueFunctions), true) ma.AddFunctions(functions, resultMetadata) } else if result, ok := res.(chunk.ReadJobResult); ok { @@ -253,11 +252,15 @@ func (ma *Aggregator) GetMetricsFromCandidates( intChunkCallTrees[i] = v i++ } - resultMetadata = utils.ExampleMetadata{ - ProfilerID: resultMetadata.ProfilerID, - ChunkID: result.Chunk.ID, - ThreadID: result.ThreadID, - } + resultMetadata = utils.NewExampleFromProfilerChunk( + result.Chunk.ProjectID, + result.Chunk.ProfilerID, + result.Chunk.ID, + result.TransactionID, + result.ThreadID, + result.Start, + result.End, + ) functions := CapAndFilterFunctions(ExtractFunctionsFromCallTrees(intChunkCallTrees), int(ma.MaxUniqueFunctions), true) ma.AddFunctions(functions, resultMetadata) } else { diff --git a/internal/nodetree/nodetree.go b/internal/nodetree/nodetree.go index 3b8d3af..23cfdcd 100644 --- a/internal/nodetree/nodetree.go +++ b/internal/nodetree/nodetree.go @@ -6,6 +6,7 @@ import ( "github.com/getsentry/vroom/internal/frame" "github.com/getsentry/vroom/internal/platform" + "github.com/getsentry/vroom/internal/utils" ) var ( @@ -38,11 +39,12 @@ type ( Package string `json:"package"` Path string `json:"path,omitempty"` - EndNS uint64 `json:"-"` - Frame frame.Frame `json:"-"` - SampleCount int `json:"-"` - StartNS uint64 `json:"-"` - ProfileIDs map[string]struct{} `json:"profile_ids,omitempty"` + EndNS uint64 `json:"-"` + Frame frame.Frame `json:"-"` + SampleCount int `json:"-"` + StartNS uint64 `json:"-"` + ProfileIDs map[string]struct{} `json:"profile_ids,omitempty"` + Profiles map[utils.ExampleMetadata]struct{} `json:"profiles,omitempty"` } ) @@ -63,6 +65,7 @@ func NodeFromFrame(f frame.Frame, start, end, fingerprint uint64) *Node { SampleCount: 1, StartNS: start, ProfileIDs: map[string]struct{}{}, + Profiles: map[utils.ExampleMetadata]struct{}{}, } if end > 0 { n.DurationNS = n.EndNS - n.StartNS diff --git a/internal/profile/android_test.go b/internal/profile/android_test.go index 506ea08..4db9f78 100644 --- a/internal/profile/android_test.go +++ b/internal/profile/android_test.go @@ -700,7 +700,7 @@ func TestCallTrees(t *testing.T) { } options := cmp.Options{ - cmpopts.IgnoreFields(nodetree.Node{}, "Fingerprint", "ProfileIDs"), + cmpopts.IgnoreFields(nodetree.Node{}, "Fingerprint", "ProfileIDs", "Profiles"), cmpopts.IgnoreFields(frame.Frame{}, "File"), } diff --git a/internal/sample/sample_test.go b/internal/sample/sample_test.go index 2268dc4..ca2b9d5 100644 --- a/internal/sample/sample_test.go +++ b/internal/sample/sample_test.go @@ -8,6 +8,7 @@ import ( "github.com/getsentry/vroom/internal/platform" "github.com/getsentry/vroom/internal/testutil" "github.com/getsentry/vroom/internal/transaction" + "github.com/getsentry/vroom/internal/utils" ) func TestReplaceIdleStacks(t *testing.T) { @@ -386,6 +387,7 @@ func TestCallTrees(t *testing.T) { StartNS: 10, Frame: frame.Frame{Function: "function0"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), Children: []*nodetree.Node{ { DurationNS: 40, @@ -397,6 +399,7 @@ func TestCallTrees(t *testing.T) { SampleCount: 2, Frame: frame.Frame{Function: "function1"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), Children: []*nodetree.Node{ { DurationNS: 10, @@ -408,6 +411,7 @@ func TestCallTrees(t *testing.T) { StartNS: 40, Frame: frame.Frame{Function: "function2"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), }, }, }, @@ -450,6 +454,7 @@ func TestCallTrees(t *testing.T) { StartNS: 10, Frame: frame.Frame{Function: "function0"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), Children: []*nodetree.Node{ { DurationNS: 30, @@ -461,6 +466,7 @@ func TestCallTrees(t *testing.T) { StartNS: 10, Frame: frame.Frame{Function: "function1"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), }, }, }, @@ -503,6 +509,7 @@ func TestCallTrees(t *testing.T) { StartNS: 10, Frame: frame.Frame{Function: "function0"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), }, { DurationNS: 10, @@ -514,6 +521,7 @@ func TestCallTrees(t *testing.T) { StartNS: 20, Frame: frame.Frame{Function: "function1"}, ProfileIDs: make(map[string]struct{}), + Profiles: make(map[utils.ExampleMetadata]struct{}), }, }, }, @@ -1255,6 +1263,7 @@ func TestCallTreesFingerprintPerPlatform(t *testing.T) { Package: "foo", SampleCount: 1, ProfileIDs: map[string]struct{}{}, + Profiles: make(map[utils.ExampleMetadata]struct{}), Frame: frame.Frame{ Data: frame.Data{SymbolicatorStatus: "symbolicated"}, Function: "main", @@ -1309,6 +1318,7 @@ func TestCallTreesFingerprintPerPlatform(t *testing.T) { Package: "foo", SampleCount: 1, ProfileIDs: map[string]struct{}{}, + Profiles: make(map[utils.ExampleMetadata]struct{}), Frame: frame.Frame{ Data: frame.Data{SymbolicatorStatus: "symbolicated"}, Function: "main", @@ -1366,6 +1376,7 @@ func TestCallTreesFingerprintPerPlatform(t *testing.T) { Path: "/usr/local/lib/python3.8/threading.py", SampleCount: 1, ProfileIDs: map[string]struct{}{}, + Profiles: make(map[utils.ExampleMetadata]struct{}), Frame: frame.Frame{ Function: "Threading.run", File: "threading.py", diff --git a/internal/speedscope/speedscope.go b/internal/speedscope/speedscope.go index 740afcb..2fa71e5 100644 --- a/internal/speedscope/speedscope.go +++ b/internal/speedscope/speedscope.go @@ -76,8 +76,9 @@ type ( } SharedData struct { - Frames []Frame `json:"frames"` - ProfileIDs []string `json:"profile_ids,omitempty"` + Frames []Frame `json:"frames"` + ProfileIDs []string `json:"profile_ids,omitempty"` + Profiles []utils.ExampleMetadata `json:"profiles,omitempty"` } EventType string diff --git a/internal/utils/structs.go b/internal/utils/structs.go index 442434c..ff2dc4f 100644 --- a/internal/utils/structs.go +++ b/internal/utils/structs.go @@ -13,12 +13,13 @@ type ( } ContinuousProfileCandidate struct { - ProjectID uint64 `json:"project_id"` - ProfilerID string `json:"profiler_id"` - ChunkID string `json:"chunk_id"` - ThreadID *string `json:"thread_id"` - Start uint64 `json:"start,string"` - End uint64 `json:"end,string"` + ProjectID uint64 `json:"project_id"` + ProfilerID string `json:"profiler_id"` + ChunkID string `json:"chunk_id"` + TransactionID string `json:"transaction_id"` + ThreadID *string `json:"thread_id"` + Start uint64 `json:"start,string"` + End uint64 `json:"end,string"` } // ExampleMetadata and FunctionMetrics have been moved here, although they'd @@ -26,10 +27,14 @@ type ( // hell that'd be introduced following the optimization to support metrics // generation within the flamegraph logic. ExampleMetadata struct { - ProfileID string `json:"profile_id,omitempty"` - ProfilerID string `json:"profiler_id,omitempty"` - ChunkID string `json:"chunk_id,omitempty"` - ThreadID *string `json:"thread_id,omitempty"` + ProjectID uint64 `json:"project_id,omitempty"` + ProfileID string `json:"profile_id,omitempty"` + ProfilerID string `json:"profiler_id,omitempty"` + ChunkID string `json:"chunk_id,omitempty"` + TransactionID string `json:"transaction_id,omitempty"` + ThreadID *string `json:"thread_id,omitempty"` + Start float64 `json:"start,omitempty"` + End float64 `json:"end,omitempty"` } FunctionMetrics struct { @@ -47,3 +52,33 @@ type ( Examples []ExampleMetadata `json:"examples"` } ) + +func NewExampleFromProfileID( + projectID uint64, + profileID string, +) ExampleMetadata { + return ExampleMetadata{ + ProjectID: projectID, + ProfileID: profileID, + } +} + +func NewExampleFromProfilerChunk( + projectID uint64, + profilerID string, + chunkID string, + transactionID string, + threadID *string, + start uint64, + end uint64, +) ExampleMetadata { + return ExampleMetadata{ + ProjectID: projectID, + ProfilerID: profilerID, + ChunkID: chunkID, + TransactionID: transactionID, + ThreadID: threadID, + Start: float64(start) / 1e9, + End: float64(end) / 1e9, + } +} From cad253f130ef4a76fdf6a64539326dd600d85b13 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Wed, 7 Aug 2024 20:27:53 -0400 Subject: [PATCH 2/5] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45e51a3..014fee4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Pass thread id to calltree generation ([#492](https://github.com/getsentry/vroom/pull/492)) - Dual mode metrics endpoint ([#493](https://github.com/getsentry/vroom/pull/493)) - Add optional generation of metrics during flamegraph aggregation ([#494](https://github.com/getsentry/vroom/pull/494)) +- Annotate flamegraph with profile data ([#501](https://github.com/getsentry/vroom/pull/501)) **Bug Fixes**: From 76c96d5f28b0e0458b3c04fd62da5f2b22709b8b Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Aug 2024 10:04:47 -0400 Subject: [PATCH 3/5] annotate chunk flamegraph with example --- internal/flamegraph/flamegraph.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/flamegraph/flamegraph.go b/internal/flamegraph/flamegraph.go index b79dcbb..4255a3a 100644 --- a/internal/flamegraph/flamegraph.go +++ b/internal/flamegraph/flamegraph.go @@ -438,9 +438,21 @@ func GetFlamegraphFromChunks( continue } intervals := []utils.Interval{interval} + + annotate := annotateWithProfileExample( + utils.NewExampleFromProfilerChunk( + result.Chunk.ProjectID, + result.Chunk.ProfilerID, + result.Chunk.ID, + result.TransactionID, + result.ThreadID, + result.Start, + result.End, + ), + ) for _, callTree := range callTrees { slicedTree := sliceCallTree(&callTree, &intervals) - addCallTreeToFlamegraph(&flamegraphTree, slicedTree, annotateWithProfileID(result.Chunk.ID)) + addCallTreeToFlamegraph(&flamegraphTree, slicedTree, annotate) } } countChunksAggregated++ From eae75469c20484cf6abcf6a8c6d60e16477a6f74 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Aug 2024 10:28:45 -0400 Subject: [PATCH 4/5] fix tests --- internal/flamegraph/flamegraph_test.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/internal/flamegraph/flamegraph_test.go b/internal/flamegraph/flamegraph_test.go index 7ce6388..7fb4bda 100644 --- a/internal/flamegraph/flamegraph_test.go +++ b/internal/flamegraph/flamegraph_test.go @@ -295,7 +295,30 @@ func TestAnnotatingWithExamples(t *testing.T) { }, } - options := cmp.Options{} + options := cmp.Options{ + // This option will order profile IDs since we only want to compare values and not order. + cmpopts.SortSlices(func(a, b utils.ExampleMetadata) bool { + if a.ProjectID != b.ProjectID { + return a.ProjectID < b.ProjectID + } + if a.ProfilerID != b.ProfilerID { + return a.ProfilerID < b.ProfilerID + } + if a.ChunkID != b.ChunkID { + return a.ChunkID < b.ChunkID + } + if a.TransactionID != b.TransactionID { + return a.TransactionID < b.TransactionID + } + if a.Start != b.Start { + return a.Start < b.Start + } + if a.End != b.End { + return a.End < b.End + } + return a.ProfileID < b.ProfileID + }), + } for _, test := range tests { t.Run(test.name, func(t *testing.T) { From c3640688418c3f883f562ffcfda11b300a3b7719 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Thu, 8 Aug 2024 14:21:02 -0400 Subject: [PATCH 5/5] expose samples example index --- internal/flamegraph/flamegraph.go | 1 + internal/flamegraph/flamegraph_test.go | 2 ++ internal/speedscope/speedscope.go | 1 + 3 files changed, 4 insertions(+) diff --git a/internal/flamegraph/flamegraph.go b/internal/flamegraph/flamegraph.go index 4255a3a..141d048 100644 --- a/internal/flamegraph/flamegraph.go +++ b/internal/flamegraph/flamegraph.go @@ -245,6 +245,7 @@ func toSpeedscope(trees []*nodetree.Node, minFreq int, projectID uint64) speedsc aggProfiles[0] = speedscope.SampledProfile{ Samples: fd.samples, SamplesProfiles: fd.samplesProfileIDs, + SamplesExamples: fd.samplesProfiles, Weights: fd.sampleCounts, SampleCounts: fd.sampleCounts, SampleDurationsNs: fd.sampleDurationsNs, diff --git a/internal/flamegraph/flamegraph_test.go b/internal/flamegraph/flamegraph_test.go index 7fb4bda..820fed4 100644 --- a/internal/flamegraph/flamegraph_test.go +++ b/internal/flamegraph/flamegraph_test.go @@ -142,6 +142,7 @@ func TestFlamegraphAggregation(t *testing.T) { {0}, {1}, }, + SamplesExamples: [][]int{{}, {}, {}, {}}, Type: "sampled", Unit: "count", Weights: []uint64{2, 1, 1, 1}, @@ -263,6 +264,7 @@ func TestAnnotatingWithExamples(t *testing.T) { {0}, }, SamplesProfiles: [][]int{{}, {}}, + SamplesExamples: [][]int{{0, 1}, {0, 1}}, Type: "sampled", Unit: "count", Weights: []uint64{2, 2}, diff --git a/internal/speedscope/speedscope.go b/internal/speedscope/speedscope.go index 2fa71e5..5824bec 100644 --- a/internal/speedscope/speedscope.go +++ b/internal/speedscope/speedscope.go @@ -65,6 +65,7 @@ type ( Queues map[string]Queue `json:"queues,omitempty"` Samples [][]int `json:"samples"` SamplesProfiles [][]int `json:"samples_profiles,omitempty"` + SamplesExamples [][]int `json:"samples_examples,omitempty"` StartValue uint64 `json:"startValue"` State string `json:"state,omitempty"` ThreadID uint64 `json:"threadID"`