Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add new 'symbolicate' commands to dotnet-stack #2436

Merged
merged 9 commits into from
Aug 25, 2021
Merged

Add new 'symbolicate' commands to dotnet-stack #2436

Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9ceee07
Add new 'symbolicate' commands to dotnet-stack
JongHeonChoi Jul 14, 2021
4f57607
Code review feedback to symbolicate in dotnet-stack
JongHeonChoi Jul 19, 2021
82e0fd3
Code review feedback
JongHeonChoi Jul 26, 2021
43f39b2
Code review feedback
JongHeonChoi Jul 29, 2021
b4d1835
Modified code using 'System.Reflection.Metadata'
JongHeonChoi Aug 5, 2021
83787af
Code review feedback
JongHeonChoi Aug 11, 2021
13f1efa
Last code review feedback
JongHeonChoi Aug 17, 2021
92a3848
Code review feedback
JongHeonChoi Aug 19, 2021
2607adb
bug fixed
JongHeonChoi Aug 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
273 changes: 151 additions & 122 deletions src/Tools/dotnet-stack/Symbolicate.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,19 @@ namespace Microsoft.Diagnostics.Tools.Stack
internal static class SymbolicateHandler
{
private static readonly Regex s_regex = new Regex(@" at (?<type>[\w+\.?]+)\.(?<method>\w+)\((?<params>.*)\) in (?<filename>[\w+\.?]+)(\.dll|\.ni\.dll): token (?<token>0x\d+)\+(?<offset>0x\d+)", RegexOptions.Compiled);
private static readonly Dictionary<string, string> s_assemblyFilePathDictionary = new Dictionary<string, string>();
private static readonly Dictionary<string, MetadataReader> s_metadataReaderDictionary = new Dictionary<string, MetadataReader>();

delegate void SymbolicateDelegate(IConsole console, FileInfo inputPath, DirectoryInfo[] searchDir, FileInfo output, bool stdout);

/// <summary>
/// Get the line number from the Method Token and IL Offset at the stacktrace
/// Get the line number from the Method Token and IL Offset in a stacktrace
/// </summary>
/// <param name="console"></param>
/// <param name="inputPath">The input path for file with stacktrace text</param>
/// <param name="searchDir">All paths in the directory to the assembly and pdb where the exception occurred</param>
/// <param name="output">The output path for the extracted line number data</param>
/// <param name="inputPath">Path to the stacktrace text file</param>
/// <param name="searchDir">Path of multiple directories with assembly and pdb where the exception occurred</param>
/// <param name="output">Output directly to a file</param>
/// <param name="stdout">Output directly to a console</param>
/// <returns></returns>
private static void Symbolicate(IConsole console, FileInfo inputPath, DirectoryInfo[] searchDir, FileInfo output, bool stdout)
{
Expand All @@ -41,135 +43,108 @@ private static void Symbolicate(IConsole console, FileInfo inputPath, DirectoryI
output = new FileInfo(inputPath.FullName + ".symbolicated");
}

CreateSymbolicateFile(console, searchDir, inputPath.FullName, output.FullName, stdout);
SetAssemblyFilePathDictionary(console, searchDir);

CreateSymbolicateFile(console, inputPath.FullName, output.FullName, stdout);
}
catch (Exception e)
{
console.Error.WriteLine(e.Message);
}
}

private static void CreateSymbolicateFile(IConsole console, DirectoryInfo[] searchDir, string inputPath, string outputPath, bool isStdout)
private static void SetAssemblyFilePathDictionary(IConsole console, DirectoryInfo[] searchDir)
{
try
{
SetMetadataReader(searchDir);
List<string> searchPaths = new List<string>
{
Directory.GetCurrentDirectory()
};
foreach (var path in searchDir)
{
searchPaths.Add(path.FullName);
}

string ret = string.Empty;
using StreamWriter fileStreamWriter = new StreamWriter(new FileStream(outputPath, FileMode.Create, FileAccess.Write));
using StreamReader fileStreamReader = new StreamReader(new FileStream(inputPath, FileMode.Open, FileAccess.Read));
while (!fileStreamReader.EndOfStream)
List<string> peFiles = GrabFiles(searchPaths, "*.dll");
if (peFiles.Count == 0)
{
string line = fileStreamReader.ReadLine();
if (!s_regex.Match(line).Success)
throw new FileNotFoundException("Assembly file not found\n");
}
peFiles = peFiles.Distinct().ToList();
peFiles.Sort();

List<string> pdbFiles = GrabFiles(searchPaths, "*.pdb");
if (pdbFiles.Count == 0)
{
throw new FileNotFoundException("PDB file not found\n");
}
pdbFiles = pdbFiles.Distinct().ToList();
pdbFiles.Sort();

int pdbCnt = 0;
for (int peCnt = 0; peCnt < peFiles.Count; peCnt++)
{
if (peFiles[peCnt].Contains(".ni.dll"))
{
fileStreamWriter?.WriteLine(ret);
if (isStdout) console.Out.WriteLine(ret);
continue;
}
ret = TrySymbolicateLine(line);
fileStreamWriter?.WriteLine(ret);
if (isStdout) console.Out.WriteLine(ret);
int compare = string.Compare(Path.GetFileNameWithoutExtension(peFiles[peCnt]), Path.GetFileNameWithoutExtension(pdbFiles[pdbCnt]), StringComparison.OrdinalIgnoreCase);
if (compare == 0)
{
s_assemblyFilePathDictionary.Add(Path.GetFileNameWithoutExtension(peFiles[peCnt]), peFiles[peCnt]);
}
else if (compare > 0)
{
pdbCnt++;
peCnt--;
}
if (pdbCnt == pdbFiles.Count) break;
}
console.Out.WriteLine($"\nOutput: {outputPath}\n");
}
catch (Exception e)
{
console.Error.WriteLine(e.Message);
}
}

private static void SetMetadataReader(DirectoryInfo[] searchDir)
private static List<string> GrabFiles(List<string> paths, string searchPattern)
{
List<string> searchPaths = new List<string>();
searchPaths.Add(Directory.GetCurrentDirectory());
foreach (var path in searchDir)
{
searchPaths.Add(path.FullName);
}

List<string> peFiles = GrabFiles(searchPaths, "*.dll");
if (peFiles.Count == 0)
{
throw new FileNotFoundException("Assembly file not found\n");
}
peFiles = peFiles.Distinct().ToList();
peFiles.Sort();

List<string> pdbFiles = GrabFiles(searchPaths, "*.pdb");
if (pdbFiles.Count == 0)
{
throw new FileNotFoundException("PDB file not found\n");
}
pdbFiles = pdbFiles.Distinct().ToList();
pdbFiles.Sort();

int pdbCnt = 0;
for (int peCnt = 0; peCnt < peFiles.Count; peCnt++)
try
{
if (peFiles[peCnt].Contains(".ni.dll"))
{
continue;
}
int compare = string.Compare(Path.GetFileNameWithoutExtension(peFiles[peCnt]), Path.GetFileNameWithoutExtension(pdbFiles[pdbCnt]), StringComparison.OrdinalIgnoreCase);
if (compare == 0)
List<string> files = new List<string>();
foreach (var assemDir in paths)
{
SetMetadataReaderDictionary(peFiles[peCnt]);
}
else if (compare > 0)
{
pdbCnt++;
peCnt--;
if (Directory.Exists(assemDir))
{
files.AddRange(Directory.GetFiles(assemDir, searchPattern, SearchOption.AllDirectories));
}
}
if (pdbCnt == pdbFiles.Count) break;
return files;
}
}

private static List<string> GrabFiles(List<string> paths, string searchPattern)
{
List<string> files = new List<string>();
foreach (var assemDir in paths)
catch
{
if (Directory.Exists(assemDir))
{
files.AddRange(Directory.GetFiles(assemDir, searchPattern, SearchOption.AllDirectories));
}
return new List<string>();
}
return files;
}

private static void SetMetadataReaderDictionary(string filePath)
private static void CreateSymbolicateFile(IConsole console, string inputPath, string outputPath, bool isStdout)
{
try
{
static Stream streamProvider(string sp) => new FileStream(sp, FileMode.Open, FileAccess.Read);
using Stream stream = new FileStream(filePath, FileMode.Open, FileAccess.Read);
if (stream != null)
using StreamWriter fileStreamWriter = new StreamWriter(new FileStream(outputPath, FileMode.Create, FileAccess.Write));
using StreamReader fileStreamReader = new StreamReader(new FileStream(inputPath, FileMode.Open, FileAccess.Read));
while (!fileStreamReader.EndOfStream)
{
MetadataReaderProvider provider = null;
if (filePath.Contains(".dll"))
{
using PEReader peReader = new PEReader(stream);
if (!peReader.TryOpenAssociatedPortablePdb(filePath, streamProvider, out provider, out string pdbPath))
{
return;
}
}
/*else if (filePath.Contains(".pdb"))
{
provider = MetadataReaderProvider.FromPortablePdbStream(stream);
}*/
else
{
return;
}
MetadataReader reader = provider?.GetMetadataReader();
s_metadataReaderDictionary.Add(Path.GetFileNameWithoutExtension(filePath), reader);
string ret = TrySymbolicateLine(fileStreamReader.ReadLine());
fileStreamWriter?.WriteLine(ret);
if (isStdout) console.Out.WriteLine(ret);
}
console.Out.WriteLine($"\nOutput: {outputPath}\n");
}
catch
catch (Exception e)
{
return;
console.Error.WriteLine(e.Message);
}
}

Expand All @@ -186,7 +161,6 @@ internal sealed class StackTraceInfo

private static string TrySymbolicateLine(string line)
{
string ret = line;
Match match = s_regex.Match(line);
if (!match.Success)
{
Expand All @@ -200,54 +174,109 @@ private static string TrySymbolicateLine(string line)
Param = match.Groups["params"].Value,
Assembly = match.Groups["filename"].Value,
Token = match.Groups["token"].Value,
Offset = match.Groups["offset"].Value
Offset = match.Groups["offset"].Value,
Pdb = match.Groups["filename"].Value + ".pdb"
};
stInfo.Pdb = stInfo.Assembly.Contains(".ni.dll") ? stInfo.Assembly.Replace(".ni.dll", ".pdb") : stInfo.Assembly.Replace(".dll", ".pdb");

return GetLineFromMetadata(TryGetMetadataReader(stInfo.Assembly), ret, stInfo);
return GetLineFromMetadata(TryGetMetadataReader(stInfo.Assembly), line, stInfo);
}

private static MetadataReader TryGetMetadataReader(string assemblyName)
{
if (s_metadataReaderDictionary.ContainsKey(assemblyName))
MetadataReader reader = null;
try
{
return s_metadataReaderDictionary[assemblyName];
if (s_assemblyFilePathDictionary.TryGetValue(assemblyName, out string filePath))
{
if (s_metadataReaderDictionary.TryGetValue(filePath, out reader))
{
return reader;
}
s_metadataReaderDictionary.Add(filePath, SetMetadataReader(filePath));
return s_metadataReaderDictionary[filePath];
}
return reader;
}
catch
{
return reader;
}
return null;
}

private static string GetLineFromMetadata(MetadataReader reader, string line, StackTraceInfo stInfo)
private static MetadataReader SetMetadataReader(string filePath)
{
if (reader != null)
MetadataReader reader = null;
try
{
Handle handle = MetadataTokens.Handle(Convert.ToInt32(stInfo.Token, 16));
if (handle.Kind == HandleKind.MethodDefinition)
MetadataReaderProvider provider = null;
static Stream streamProvider(string sp) => new FileStream(sp, FileMode.Open, FileAccess.Read);
using Stream stream = new FileStream(filePath, FileMode.Open, FileAccess.Read);
if (stream != null)
{
MethodDebugInformationHandle methodDebugHandle = ((MethodDefinitionHandle)handle).ToDebugInformationHandle();
MethodDebugInformation methodInfo = reader.GetMethodDebugInformation(methodDebugHandle);
if (!methodInfo.SequencePointsBlob.IsNil)
if (filePath.Contains(".dll"))
{
SequencePointCollection sequencePoints = methodInfo.GetSequencePoints();
SequencePoint? bestPointSoFar = null;
foreach (SequencePoint point in sequencePoints)
using PEReader peReader = new PEReader(stream);
if (!peReader.TryOpenAssociatedPortablePdb(filePath, streamProvider, out provider, out string pdbPath))
{
if (point.Offset > Convert.ToInt64(stInfo.Offset, 16))
break;

if (point.StartLine != SequencePoint.HiddenLine)
bestPointSoFar = point;
return reader;
}
}
/*else if (filePath.Contains(".pdb"))
{
provider = MetadataReaderProvider.FromPortablePdbStream(stream);
}*/
else
{
return reader;
}
}
return provider?.GetMetadataReader();
}
catch
{
return reader;
}
}

if (bestPointSoFar.HasValue)
private static string GetLineFromMetadata(MetadataReader reader, string line, StackTraceInfo stInfo)
{
try
{
if (reader != null)
{
Handle handle = MetadataTokens.Handle(Convert.ToInt32(stInfo.Token, 16));
if (handle.Kind == HandleKind.MethodDefinition)
{
MethodDebugInformationHandle methodDebugHandle = ((MethodDefinitionHandle)handle).ToDebugInformationHandle();
MethodDebugInformation methodInfo = reader.GetMethodDebugInformation(methodDebugHandle);
if (!methodInfo.SequencePointsBlob.IsNil)
{
string sourceFile = reader.GetString(reader.GetDocument(bestPointSoFar.Value.Document).Name);
int sourceLine = bestPointSoFar.Value.StartLine;
return $" at {stInfo.Type}.{stInfo.Method}({stInfo.Param}) in {sourceFile}:line {sourceLine}";
SequencePointCollection sequencePoints = methodInfo.GetSequencePoints();
SequencePoint? bestPointSoFar = null;
foreach (SequencePoint point in sequencePoints)
{
if (point.Offset > Convert.ToInt64(stInfo.Offset, 16))
break;

if (point.StartLine != SequencePoint.HiddenLine)
bestPointSoFar = point;
}

if (bestPointSoFar.HasValue)
{
string sourceFile = reader.GetString(reader.GetDocument(bestPointSoFar.Value.Document).Name);
int sourceLine = bestPointSoFar.Value.StartLine;
return $" at {stInfo.Type}.{stInfo.Method}({stInfo.Param}) in {sourceFile}:line {sourceLine}";
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Theoretically, the input line could have any number or type of whitespace (or other characters) at the beginning of the line. For example, the input text may be from a log that has timestamps on every line. We could potentially be throwing out data by rewriting the line like this.

If possible, we should try to rewrite the line in situ. You could reuse the regex you already have with the Regex.Replace(String, String, MatchEvaluator, RegexOptions) API. You would write a MatchEvaluator method that receives each match in the regex and replaces it with the appropriate part of the StackTraceInfo. You may also need to add a capture group for the : token part so you can replace it with : line.

}
}
}
}
return line;
}
catch
{
return line;
}
return line;
}

public static Command SymbolicateCommand() =>
Expand Down Expand Up @@ -285,7 +314,7 @@ public static Option<FileInfo> OutputFileOption() =>
Argument = new Argument<FileInfo>(name: "output-path")
{
Arity = ArgumentArity.ZeroOrOne
}
}.ExistingOnly()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not familiar with this method. Is this saying that the output file needs to exist in order for the option to be valid? That seems opposite of the intention of the option, i.e., create the file and fill it with the contents we generate.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this will generate an error if the file doesn't exist.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh. It's my mistake. I'll delete it.

};

public static Option<bool> StandardOutOption() =>
Expand Down