Use PEImage API get export symbols for single file apps (#2674)

* Use PEImage API get export symbols for single file apps

Add single file app testing support.

Disable bpmd testing for single-file on Linux/MacOS

Issues: #2515 and #2438

* Support Windows/MacOS single-file in the C++ runtime enum fallback

IDebuggerServices::GetOffsetBySymbol is used to get the runtime info export and so most
of the changes are making the interface available on the libsos "side" under Linux/MacOS
since there is a Extensions instance both in libsosplugin and libsos. The libsos side
Extensions::GetDebuggerServices always returned null before this change. Now it only
returns null if hosted under a managed host like dotnet-dump.

* Disable test single-file live apps on Alpine

* Enable single-file tests when no host runtime (C++ fallback)

* Fix initialize bpmd problems on Main

* Fix clrstack -i/ICorDebug C++ fallback code for single-file

Simplify to what the managed Runtime code does without the debugshim
and library provider. SOS no longer needs debugshim, but left it for
the future dbgshim out-of-band work.

src/Microsoft.Diagnostics.DebugServices.Implementation/Module.cs

@@ -4,11 +4,14 @@
 
 using Microsoft.Diagnostics.Runtime;
 using Microsoft.Diagnostics.Runtime.Utilities;
+using Microsoft.FileFormats;
 using Microsoft.FileFormats.ELF;
 using Microsoft.FileFormats.MachO;
+using Microsoft.FileFormats.PE;
 using System;
 using System.Collections.Immutable;
 using System.Diagnostics;
+using System.IO;
 using System.Reflection.PortableExecutable;
 using System.Runtime.InteropServices;
 using FileVersionInfo = Microsoft.Diagnostics.Runtime.Utilities.FileVersionInfo;
@@ -18,7 +21,7 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation
     /// <summary>
     /// Module base implementation
     /// </summary>
-    public abstract class Module : IModule, IDisposable
+    public abstract class Module : IModule, IExportSymbols, IDisposable
     {
         [Flags]
         public enum Flags : byte
@@ -58,6 +61,7 @@ public Module(ITarget target)
                 ServiceProvider.RemoveService(typeof(ELFFile));
                 ServiceProvider.RemoveService(typeof(PEReader));
             });
+            ServiceProvider.AddService<IExportSymbols>(this);
          }
 
         public void Dispose()
@@ -163,6 +167,57 @@ public ImmutableArray<byte> BuildId
 
         #endregion
 
+        #region IExportSymbols
+
+        bool IExportSymbols.TryGetSymbolAddress(string name, out ulong address)
+        {
+            if (Target.OperatingSystem == OSPlatform.Windows)
+            {
+                Stream stream = Target.Services.GetService<IMemoryService>().CreateMemoryStream(ImageBase, ImageSize);
+                PEFile image = new(new StreamAddressSpace(stream), isDataSourceVirtualAddressSpace: true);
+                if (image.IsValid())
+                { 
+                    if (image.TryGetExportSymbol(name, out ulong offset))
+                    {
+                        address = ImageBase + offset;
+                        return true;
+                    }
+                    address = 0;
+                    return false;
+                }
+            }
+            else if (Target.OperatingSystem == OSPlatform.Linux)
+            {
+                try
+                {
+                    Stream stream = Target.Services.GetService<IMemoryService>().CreateMemoryStream(ImageBase, ImageSize);
+                    ElfFile elfFile = new(stream, position: ImageBase, leaveOpen: false, isVirtual: true);
+                    if (elfFile.Header.IsValid)
+                    {
+                        if (elfFile.TryGetExportSymbol(name, out ulong offset))
+                        {
+                            address = ImageBase + offset;
+                            return true;
+                        }
+                        address = 0;
+                        return false;
+                    }
+                }
+                catch (InvalidDataException)
+                {
+                }
+            }
+            return TryGetSymbolAddressInner(name, out address);
+        }
+
+        protected virtual bool TryGetSymbolAddressInner(string name, out ulong address)
+        {
+            address = 0;
+            return false;
+        }
+
+        #endregion
+
         protected VersionData GetVersion()
         {
             VersionData versionData = null;

src/Microsoft.Diagnostics.DebugServices.Implementation/ModuleServiceFromDataReader.cs

@@ -17,7 +17,7 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation
     /// </summary>
     public class ModuleServiceFromDataReader : ModuleService
     {
-        class ModuleFromDataReader : Module, IExportSymbols
+        class ModuleFromDataReader : Module
         {
             // This is what clrmd returns for non-PE modules that don't have a timestamp
             private const uint InvalidTimeStamp = 0;
@@ -38,10 +38,6 @@ public ModuleFromDataReader(ModuleServiceFromDataReader moduleService, IExportRe
                 _imageSize = imageSize;
                 _exportReader = exportReader;
                 ModuleIndex = moduleIndex;
-                if (exportReader is not null)
-                {
-                    ServiceProvider.AddService<IExportSymbols>(this);
-                }
             }
 
             #region IModule
@@ -97,28 +93,16 @@ public override string VersionString
 
             #endregion
 
-            #region IExportSymbols
-
-            public bool TryGetSymbolAddress(string name, out ulong address)
+            protected override bool TryGetSymbolAddressInner(string name, out ulong address)
             {
                 if (_exportReader is not null)
                 {
-                    // Some exceptions are escaping from the clrmd ELF dump reader. This will be
-                    // fixed in a clrmd update.
-                    try
-                    {
-                        return _exportReader.TryGetSymbolAddress(ImageBase, name, out address);
-                    }
-                    catch (IOException)
-                    {
-                    }
+                    return _exportReader.TryGetSymbolAddress(ImageBase, name, out address);
                 }
                 address = 0;
                 return false;
             }
 
-            #endregion
-
             protected override ModuleService ModuleService => _moduleService;
         }
 

src/Microsoft.Diagnostics.DebugServices.Implementation/RuntimeService.cs

@@ -21,7 +21,6 @@ namespace Microsoft.Diagnostics.DebugServices.Implementation
     public class RuntimeService : IRuntimeService, IDataReader, IExportReader
     {
         private readonly ITarget _target;
-        private readonly bool _exportReaderEnabled;
         private readonly IDisposable _onFlushEvent;
         private DataTarget _dataTarget;
         private List<Runtime> _runtimes;
@@ -33,8 +32,6 @@ public class RuntimeService : IRuntimeService, IDataReader, IExportReader
         public RuntimeService(ITarget target)
         {
             _target = target;
-            // TODO - mikem 4/30/21 - remove when dbgeng services doesn't take so long looking up exports (attempts to load PDBs).
-            _exportReaderEnabled = target.Host.HostType != HostType.DbgEng || Environment.GetEnvironmentVariable("DOTNET_ENABLE_SOS_SINGLEFILE") == "1";
             _onFlushEvent = target.OnFlushEvent.Register(() => {
                 if (_runtimes is not null && _runtimes.Count == 0)
                 {
@@ -219,20 +216,17 @@ ulong IMemoryReader.ReadPointer(ulong address)
 
         bool IExportReader.TryGetSymbolAddress(ulong baseAddress, string name, out ulong offset)
         {
-            if (_exportReaderEnabled)
+            try
             {
-                try
-                {
-                    IExportSymbols exportSymbols = ModuleService.GetModuleFromBaseAddress(baseAddress).Services.GetService<IExportSymbols>();
-                    if (exportSymbols is not null)
-                    {
-                        return exportSymbols.TryGetSymbolAddress(name, out offset);
-                    }
-                }
-                catch (DiagnosticsException)
+                IExportSymbols exportSymbols = ModuleService.GetModuleFromBaseAddress(baseAddress).Services.GetService<IExportSymbols>();
+                if (exportSymbols is not null)
                 {
+                    return exportSymbols.TryGetSymbolAddress(name, out offset);
                 }
             }
+            catch (DiagnosticsException)
+            {
+            }
             offset = 0;
             return false;
         }

src/Microsoft.Diagnostics.DebugServices.Implementation/SymbolService.cs

@@ -28,8 +28,8 @@ public class SymbolService : ISymbolService
         /// <summary>
         /// Symbol server URLs
         /// </summary>
-        public const string MsdlSymbolServer = "http://msdl.microsoft.com/download/symbols/";
-        public const string SymwebSymbolServer = "http://symweb.corp.microsoft.com/";
+        public const string MsdlSymbolServer = "https://msdl.microsoft.com/download/symbols/";
+        public const string SymwebSymbolServer = "https://symweb.corp.microsoft.com/";
 
         private readonly IHost _host;
         private string _defaultSymbolCache;

src/Microsoft.Diagnostics.TestHelpers/BaseDebuggeeCompiler.cs

@@ -165,12 +165,12 @@ protected virtual string GetDebuggeeBinaryDirPath(string debuggeeProjectDirPath,
 
         protected static string GetDebuggeeBinaryDllPath(TestConfiguration config, string debuggeeBinaryDirPath, string debuggeeName)
         {
-            return config.IsNETCore ? Path.Combine(debuggeeBinaryDirPath, debuggeeName + (config.PublishSingleFile ? "" : ".dll")) : null;
+            return config.IsNETCore ? Path.Combine(debuggeeBinaryDirPath, debuggeeName + ".dll") : null;
         }
 
         protected static string GetDebuggeeBinaryExePath(TestConfiguration config, string debuggeeBinaryDirPath, string debuggeeName)
         {
-            return config.IsDesktop ? Path.Combine(debuggeeBinaryDirPath, debuggeeName + ".exe") : null;
+            return config.IsDesktop || config.PublishSingleFile ? Path.Combine(debuggeeBinaryDirPath, debuggeeName + (OS.Kind == OSKind.Windows ? ".exe" : "")) : null;
         }
 
         protected static string GetLogPath(TestConfiguration config, string framework, string runtime, string debuggeeName)

src/Microsoft.Diagnostics.TestHelpers/CliDebuggeeCompiler.cs

@@ -38,6 +38,7 @@ private static Dictionary<string,string> GetBuildProperties(TestConfiguration co
             {
                 Debug.Assert(runtimeIdentifier != null);
                 buildProperties.Add("PublishSingleFile", "true");
+                buildProperties.Add("SelfContained", "true");
             }
             string debugType = config.DebugType;
             if (debugType == null)
@@ -78,6 +79,7 @@ public override DotNetBuildDebuggeeTestStep ConfigureDotNetBuildDebuggeeTask(Tes
                                                GetDebuggeeNativeLibDirPath(config, debuggeeName),
                                                GetBuildProperties(config, runtimeIdentifier),
                                                runtimeIdentifier,
+                                               config.BuildProjectFramework,
                                                config.LinkerPackageVersion,
                                                debuggeeName,
                                                debuggeeSolutionDirPath,

src/Microsoft.Diagnostics.TestHelpers/CsprojBuildDebuggeeTestStep.cs

@@ -30,6 +30,7 @@ public CsprojBuildDebuggeeTestStep(string dotnetToolPath,
                                        string debuggeeNativeLibDirPath,
                                        Dictionary<string,string> buildProperties,
                                        string runtimeIdentifier,
+                                       string runtimeFramework,
                                        string linkerPackageVersion,
                 					   string debuggeeName,
                                        string debuggeeSolutionDirPath,
@@ -54,6 +55,7 @@ public CsprojBuildDebuggeeTestStep(string dotnetToolPath,
         {
             BuildProperties = buildProperties;
             RuntimeIdentifier = runtimeIdentifier;
+            RuntimeFramework = runtimeFramework;
             DebuggeeName = debuggeeName;
             ProjectTemplateFileName = debuggeeName + ".csproj";
             LinkerPackageVersion = linkerPackageVersion;
@@ -64,6 +66,7 @@ public CsprojBuildDebuggeeTestStep(string dotnetToolPath,
         /// </summary>
         public IDictionary<string,string> BuildProperties { get; }
         public string RuntimeIdentifier { get; }
+        public string RuntimeFramework { get; }
         public string DebuggeeName { get; }
         public string LinkerPackageVersion { get; }
         public override string ProjectTemplateFileName { get; }
@@ -73,7 +76,7 @@ protected override async Task Restore(ITestOutputHelper output)
             string extraArgs = "";
             if (RuntimeIdentifier != null)
             {
-                extraArgs = " --runtime " + RuntimeIdentifier;
+                extraArgs += " --runtime " + RuntimeIdentifier;
             }
             foreach (var prop in BuildProperties)
             {
@@ -85,9 +88,14 @@ protected override async Task Restore(ITestOutputHelper output)
         protected override async Task Build(ITestOutputHelper output)
         {
             string publishArgs = "publish";
+            if (RuntimeFramework != null)
+            {
+                publishArgs += " --framework " + RuntimeFramework;
+            }
             if (RuntimeIdentifier != null)
             {
                 publishArgs += " --runtime " + RuntimeIdentifier;
+                publishArgs += " --self-contained true";
             }
             foreach (var prop in BuildProperties)
             {

src/SOS/CMakeLists.txt

@@ -38,5 +38,3 @@ if(CLR_CMAKE_PLATFORM_UNIX)
   # This prevents inclusion of standard C compiler headers
   add_compile_options(-nostdinc)
 endif(CLR_CMAKE_PLATFORM_UNIX)
-
-add_subdirectory(debugshim)

src/SOS/SOS.Extensions/ModuleServiceFromDebuggerServices.cs

@@ -17,7 +17,7 @@ namespace SOS.Extensions
     /// </summary>
     internal class ModuleServiceFromDebuggerServices : ModuleService
     {
-        class ModuleFromDebuggerServices : Module, IExportSymbols, IModuleSymbols
+        class ModuleFromDebuggerServices : Module, IModuleSymbols
         {
             // This is what dbgeng/IDebuggerServices returns for non-PE modules that don't have a timestamp
             private const uint InvalidTimeStamp = 0xFFFFFFFE;
@@ -44,7 +44,6 @@ public ModuleFromDebuggerServices(
                 IndexFileSize = indexTimeStamp == InvalidTimeStamp ? null : indexFileSize;
                 IndexTimeStamp = indexTimeStamp == InvalidTimeStamp ? null : indexTimeStamp;
 
-                ServiceProvider.AddService<IExportSymbols>(this);
                 ServiceProvider.AddService<IModuleSymbols>(this);
             }
 
@@ -110,20 +109,25 @@ public override string VersionString
 
             #endregion
 
-            #region IExportSymbols/IModuleSymbols
+            #region IModuleSymbols
 
-            public bool TryGetSymbolName(ulong address, out string symbol, out ulong displacement)
+            bool IModuleSymbols.TryGetSymbolName(ulong address, out string symbol, out ulong displacement)
             {
                 return _moduleService._debuggerServices.GetSymbolByOffset(ModuleIndex, address, out symbol, out displacement) == HResult.S_OK;
             }
 
-            public bool TryGetSymbolAddress(string name, out ulong address)
+            bool IModuleSymbols.TryGetSymbolAddress(string name, out ulong address)
             {
                 return _moduleService._debuggerServices.GetOffsetBySymbol(ModuleIndex, name, out address) == HResult.S_OK;
             }
 
             #endregion
 
+            protected override bool TryGetSymbolAddressInner(string name, out ulong address)
+            {
+                return _moduleService._debuggerServices.GetOffsetBySymbol(ModuleIndex, name, out address) == HResult.S_OK;
+            }
+
             protected override ModuleService ModuleService => _moduleService;
         }
 

src/SOS/SOS.Hosting/SOSLibrary.cs

@@ -24,7 +24,8 @@ private delegate int SOSCommandDelegate(
 
         [UnmanagedFunctionPointer(CallingConvention.Winapi)]
         private delegate int SOSInitializeDelegate(
-            IntPtr IHost);
+            IntPtr IHost,
+            IntPtr IDebuggerServices);
 
         [UnmanagedFunctionPointer(CallingConvention.Winapi)]
         private delegate void SOSUninitializeDelegate();
@@ -120,7 +121,7 @@ private void Initialize()
                 {
                     throw new EntryPointNotFoundException($"Can not find SOS module initialization function: {SOSInitialize}");
                 }
-                int result = initializeFunc(_hostWrapper.IHost);
+                int result = initializeFunc(_hostWrapper.IHost, IntPtr.Zero);
                 if (result != 0)
                 {
                     throw new InvalidOperationException($"SOS initialization FAILED 0x{result:X8}");