From b2bc3dd29a4e75765d263b65e784cb5e085fa977 Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Mon, 20 Jan 2020 22:31:47 -0600 Subject: [PATCH] Add HeapSnapshotCompare project This adds a C# program that compares and summarizes ETW heap snapshots. Heap snapshots are described here: https://randomascii.wordpress.com/2019/10/27/heap-snapshots-tracing-all-heap-allocations/ TraceProcessor for programmatic analysis of ETW traces is described here: https://randomascii.wordpress.com/2020/01/05/bulk-etw-trace-analysis-in-c/ --- .../HeapSnapshotCompare/App.config | 6 + .../HeapSnapshotCompare.cs | 250 ++++++++++++++++++ .../HeapSnapshotCompare.csproj | 100 +++++++ .../Properties/AssemblyInfo.cs | 36 +++ .../HeapSnapshotCompare/packages.config | 7 + TraceProcessors/TraceProcessors.sln | 14 + 6 files changed, 413 insertions(+) create mode 100644 TraceProcessors/HeapSnapshotCompare/App.config create mode 100644 TraceProcessors/HeapSnapshotCompare/HeapSnapshotCompare.cs create mode 100644 TraceProcessors/HeapSnapshotCompare/HeapSnapshotCompare.csproj create mode 100644 TraceProcessors/HeapSnapshotCompare/Properties/AssemblyInfo.cs create mode 100644 TraceProcessors/HeapSnapshotCompare/packages.config diff --git a/TraceProcessors/HeapSnapshotCompare/App.config b/TraceProcessors/HeapSnapshotCompare/App.config new file mode 100644 index 00000000..56efbc7b --- /dev/null +++ b/TraceProcessors/HeapSnapshotCompare/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/TraceProcessors/HeapSnapshotCompare/HeapSnapshotCompare.cs b/TraceProcessors/HeapSnapshotCompare/HeapSnapshotCompare.cs new file mode 100644 index 00000000..42dcde12 --- /dev/null +++ b/TraceProcessors/HeapSnapshotCompare/HeapSnapshotCompare.cs @@ -0,0 +1,250 @@ +/* +Copyright 2019 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This program summarizes one or more heap snapshots. If multiple heap +// snapshots are given as parameters then it tries to summarizes the +// differences. See this post for discussion of recording heap snapshots: +// https://randomascii.wordpress.com/2019/10/27/heap-snapshots-tracing-all-heap-allocations/ + +// See these blog posts for details of the Trace Processor package used to +// drive this: +// https://blogs.windows.com/windowsdeveloper/2019/05/09/announcing-traceprocessor-preview-0-1-0/ +// https://blogs.windows.com/windowsdeveloper/2019/08/07/traceprocessor-0-2-0/#L2W90BVvLzJ8XwEY.97 +// https://randomascii.wordpress.com/2020/01/05/bulk-etw-trace-analysis-in-c/ +// This uses the Microsoft.Windows.EventTracing.Processing.All package from NuGet + +using Microsoft.Windows.EventTracing; +using Microsoft.Windows.EventTracing.Memory; +using Microsoft.Windows.EventTracing.Symbols; +using System; +using System.Collections.Generic; +using System.IO; + +namespace HeapSnapshotCompare +{ + // A summary of a particular allocation call stack including the outstanding + // bytes allocated, the number of outstanding allocations, and the call + // stack. + struct AllocDetails + { + public DataSize Size; + public long Count; + public IReadOnlyList Stack; + } + + // A summary of a heap snapshot. This includes AllocDetails by Stack Ref #, + // which stack frames show up in the most allocation stacks, and the total + // number of bytes and allocations outstanding. + class SnapshotSummary + { + public SnapshotSummary(Dictionary allocsByStackId, ulong pid) + { + allocsByStackId_ = allocsByStackId; + pid_ = pid; + } + // Dictionary of AllocDetails indexed by SnapshotUniqueStackId, aka the Stack Ref# + // column. + public Dictionary allocsByStackId_; + public Dictionary hotStackFrames_ = null; + public ulong pid_; + + public DataSize totalBytes_; + public long allocCount_; + } + + class HeapSnapshotCompare + { + // Process a trace and create a summary. + static SnapshotSummary GetAllocSummary(ITraceProcessor trace) + { + var pendingSnapshotData = trace.UseHeapSnapshots(); + IPendingResult pendingSymbols = null; + pendingSymbols = trace.UseSymbols(); + trace.Process(); + + var snapshotData = pendingSnapshotData.Result; + + ISymbolDataSource symbols = null; + symbols = pendingSymbols.Result; + symbols.LoadSymbolsAsync(new SymCachePath(@"c:\symcache")).GetAwaiter().GetResult(); + + if (snapshotData.Snapshots.Count != 1) + { + Console.Error.WriteLine("Trace must contain exactly one heap snapshot - actually contained {0}.", + snapshotData.Snapshots.Count); + return new SnapshotSummary(null, 0); + } + + // Scan through all of the allocations and collect them by + // SnapshotUniqueStackId (which corresponds to Stack Ref#), + // accumulating the bytes allocated, allocation count, and the + // stack. + var allocsByStackId = new Dictionary(); + long highestCount = 0; + var highestCountDetails = new AllocDetails(); + foreach (IHeapAllocation row in snapshotData.Snapshots[0].Allocations) + { + allocsByStackId.TryGetValue(row.SnapshotUniqueStackId, out AllocDetails value); + value.Stack = row.Stack; + value.Size += row.Size; + value.Count += 1; + if (value.Count > highestCount) + { + highestCount = value.Count; + highestCountDetails = value; + } + allocsByStackId[row.SnapshotUniqueStackId] = value; + } + + // Count how many allocations each stack frame is part of. + // RtlThreadStart will presumably be near the top, along with + // RtlpAllocateHeapInternal, but some clues may be found. + var hotStackFrames = new Dictionary(); + foreach (var data in allocsByStackId.Values) + { + foreach (var entry in data.Stack) + { + var analyzerString = entry.GetAnalyzerString(); + hotStackFrames.TryGetValue(analyzerString, out long count); + count += data.Count; + hotStackFrames[analyzerString] = count; + } + } + + var result = new SnapshotSummary(allocsByStackId, snapshotData.Snapshots[0].ProcessId); + + // Create a summary of the alloc counts and byte counts. + var totalAllocBytes = DataSize.Zero; + long totalAllocCount = 0; + foreach (var data in allocsByStackId.Values) + { + totalAllocBytes += data.Size; + totalAllocCount += data.Count; + } + + result.hotStackFrames_ = hotStackFrames; + result.totalBytes_ = totalAllocBytes; + result.allocCount_ = totalAllocCount; + + return result; + } + + static void Main(string[] args) + { + if (args.Length == 0) + { + Console.WriteLine("Use this to summarize a heap snapshot or compare multiple heap snapshots"); + Console.WriteLine("from one run of a program."); + return; + } + + SnapshotSummary lastAllocs = null; + string lastTracename = ""; + foreach (var arg in args) + { + if (!File.Exists(arg)) + { + Console.Error.WriteLine("File '{0}' does not exist.", arg); + continue; + } + using (ITraceProcessor trace = TraceProcessor.Create(arg)) + { + Console.WriteLine("Summarizing '{0}'", Path.GetFileName(arg)); + var allocs = GetAllocSummary(trace); + if (allocs.allocsByStackId_ == null) + { + Console.WriteLine("Ignoring trace {0}.", arg); + continue; + } + Console.WriteLine("{0,7:F2} MB from {1,9:#,#} allocations on {2,7:#,#} stacks", + allocs.totalBytes_.TotalMegabytes, allocs.allocCount_, allocs.allocsByStackId_.Count); + + const int maxPrinted = 40; + + Console.WriteLine("Hottest stack frames:"); + // Display a summary of the first (possibly only) heap snapshot trace. + var sortedHotStackEntries = new List>(allocs.hotStackFrames_); + sortedHotStackEntries.Sort((x, y) => y.Value.CompareTo(x.Value)); + for (int i = 0; i < sortedHotStackEntries.Count && i < maxPrinted; ++i) + { + var data = sortedHotStackEntries[i]; + Console.WriteLine("{0,5} allocs cross {1}", data.Value, data.Key); + } + + if (lastAllocs != null) + { + Console.WriteLine("Comparing old ({0}) to new ({1}) snapshots.", Path.GetFileName(lastTracename), Path.GetFileName(arg)); + if (allocs.pid_ != lastAllocs.pid_) + { + Console.WriteLine("WARNING: process IDs are different ({0} and {1}) so stack IDs may not be comparable.", lastAllocs.pid_, allocs.pid_); + } + + var hotStackFramesDelta = new Dictionary(allocs.hotStackFrames_); + // Subtract the lastAllocs stack frame counts fomr the current stack frame counts. + foreach (var entry in lastAllocs.hotStackFrames_) + { + hotStackFramesDelta.TryGetValue(entry.Key, out long count); + count -= entry.Value; + hotStackFramesDelta[entry.Key] = count; + } + + Console.WriteLine("Hottest stack frame deltas:"); + // Print the biggest deltas, positive then negative. + var sortedHotStackFramesDelta = new List>(hotStackFramesDelta); + sortedHotStackFramesDelta.Sort((x, y) => y.Value.CompareTo(x.Value)); + // Print the first half... + for (int i = 0; i < sortedHotStackFramesDelta.Count && i < maxPrinted / 2; ++i) + { + var data = sortedHotStackFramesDelta[i]; + Console.WriteLine("{0,5} allocs cross {1}", data.Value, data.Key); + } + Console.WriteLine("..."); + int start = sortedHotStackFramesDelta.Count - maxPrinted / 2; + if (start < 0) + start = 0; + for (int i = start; i < sortedHotStackFramesDelta.Count - 1; ++i) + { + var data = sortedHotStackFramesDelta[i]; + Console.WriteLine("{0,5} allocs cross {1}", data.Value, data.Key); + } + + ulong newOnlyStacks = 0; + ulong oldOnlyStacks = 0; + foreach (var tag in allocs.allocsByStackId_.Keys) + { + if (!lastAllocs.allocsByStackId_.ContainsKey(tag)) + { + newOnlyStacks++; + } + } + foreach (var tag in lastAllocs.allocsByStackId_.Keys) + { + if (!allocs.allocsByStackId_.ContainsKey(tag)) + { + oldOnlyStacks++; + } + } + Console.WriteLine(" Old snapshot had {0} unique-to-it stacks, new trace had {1} unique-to-it stacks.", + oldOnlyStacks, newOnlyStacks); + } + + lastAllocs = allocs; + lastTracename = arg; + } + } + } + } +} diff --git a/TraceProcessors/HeapSnapshotCompare/HeapSnapshotCompare.csproj b/TraceProcessors/HeapSnapshotCompare/HeapSnapshotCompare.csproj new file mode 100644 index 00000000..a65820fd --- /dev/null +++ b/TraceProcessors/HeapSnapshotCompare/HeapSnapshotCompare.csproj @@ -0,0 +1,100 @@ + + + + + Debug + AnyCPU + {96201BE4-B511-49A3-A98D-537B77E054C6} + Exe + HeapSnapshotCompare + HeapSnapshotCompare + v4.7.2 + 512 + true + true + + + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + false + true + true + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + false + + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.Cpu.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.GenericEvents.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.HyperV.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.Interop.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.Power.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.Processing.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.Processing.Community.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.ScheduledTasks.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.Syscalls.dll + + + ..\packages\Microsoft.Windows.EventTracing.Processing.All.0.2.1\lib\netstandard2.0\Microsoft.Windows.EventTracing.WindowInFocus.dll + + + + + ..\packages\System.Security.Principal.Windows.4.4.1\lib\net461\System.Security.Principal.Windows.dll + + + + + + + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + \ No newline at end of file diff --git a/TraceProcessors/HeapSnapshotCompare/Properties/AssemblyInfo.cs b/TraceProcessors/HeapSnapshotCompare/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..e9301c71 --- /dev/null +++ b/TraceProcessors/HeapSnapshotCompare/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("HeapSnapshotCompare")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("HeapSnapshotCompare")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("96201be4-b511-49a3-a98d-537b77e054c6")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/TraceProcessors/HeapSnapshotCompare/packages.config b/TraceProcessors/HeapSnapshotCompare/packages.config new file mode 100644 index 00000000..04ed862b --- /dev/null +++ b/TraceProcessors/HeapSnapshotCompare/packages.config @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/TraceProcessors/TraceProcessors.sln b/TraceProcessors/TraceProcessors.sln index 091026a5..5e7281e6 100644 --- a/TraceProcessors/TraceProcessors.sln +++ b/TraceProcessors/TraceProcessors.sln @@ -7,6 +7,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IdentifyChromeProcesses", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CPUSummary", "CPUSummary\CPUSummary.csproj", "{C670C694-74AF-46AE-9861-005B14510DB4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeapSnapshotCompare", "HeapSnapshotCompare\HeapSnapshotCompare.csproj", "{96201BE4-B511-49A3-A98D-537B77E054C6}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -41,6 +43,18 @@ Global {C670C694-74AF-46AE-9861-005B14510DB4}.Release|x64.Build.0 = Release|Any CPU {C670C694-74AF-46AE-9861-005B14510DB4}.Release|x86.ActiveCfg = Release|Any CPU {C670C694-74AF-46AE-9861-005B14510DB4}.Release|x86.Build.0 = Release|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Debug|x64.ActiveCfg = Debug|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Debug|x64.Build.0 = Debug|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Debug|x86.ActiveCfg = Debug|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Debug|x86.Build.0 = Debug|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Release|Any CPU.Build.0 = Release|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Release|x64.ActiveCfg = Release|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Release|x64.Build.0 = Release|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Release|x86.ActiveCfg = Release|Any CPU + {96201BE4-B511-49A3-A98D-537B77E054C6}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE