From 410b90d2fbc84370ace0c5d601470cfb7cd56224 Mon Sep 17 00:00:00 2001 From: Washi Date: Thu, 16 Jun 2022 20:34:55 +0200 Subject: [PATCH 01/16] Basic read support PDB info stream. --- .../Metadata/Info/InfoStream.cs | 124 ++++++++++++++++++ .../Metadata/Info/InfoStreamVersion.cs | 19 +++ .../Metadata/Info/PdbFeature.cs | 28 ++++ .../Metadata/Info/SerializedInfoStream.cs | 83 ++++++++++++ .../Metadata/PdbHashTable.cs | 36 +++++ .../Metadata/Info/InfoStreamTest.cs | 30 +++++ 6 files changed, 320 insertions(+) create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStreamVersion.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Info/PdbFeature.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs create mode 100644 test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs new file mode 100644 index 000000000..182981fe1 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using AsmResolver.IO; + +namespace AsmResolver.Symbols.Pdb.Metadata.Info; + +/// +/// Represents the PDB Info Stream (also known as the PDB stream) +/// +public class InfoStream : SegmentBase +{ + private IDictionary? _streamIndices; + private IList? _features; + + /// + /// Gets or sets the version of the file format of the PDB info stream. + /// + /// + /// Modern tooling only recognize the VC7.0 file format. + /// + public InfoStreamVersion Version + { + get; + set; + } + + /// + /// Gets or sets the 32-bit UNIX time-stamp of the PDB file. + /// + public uint Signature + { + get; + set; + } + + /// + /// Gets or sets the number of times the PDB file has been written. + /// + public uint Age + { + get; + set; + } + + /// + /// Gets or sets the unique identifier assigned to the PDB file. + /// + public Guid UniqueId + { + get; + set; + } + + /// + /// Gets a mapping from stream names to their respective stream index within the underlying MSF file. + /// + public IDictionary StreamIndices + { + get + { + if (_streamIndices is null) + Interlocked.CompareExchange(ref _streamIndices, GetStreamIndices(), null); + return _streamIndices; + } + } + + /// + /// Gets a list of characteristics that this PDB has. + /// + public IList Features + { + get + { + if (_features is null) + Interlocked.CompareExchange(ref _features, GetFeatures(), null); + return _features; + } + } + + /// + /// Reads a single PDB info stream from the provided input stream. + /// + /// The input stream. + /// The parsed info stream. + public static InfoStream FromReader(BinaryStreamReader reader) => new SerializedInfoStream(reader); + + /// + /// Obtains the stream name to index mapping of the PDB file. + /// + /// The mapping. + /// + /// This method is called upon initialization of the property. + /// + protected virtual IDictionary GetStreamIndices() => new Dictionary(); + + /// + /// Obtains the features of the PDB file. + /// + /// The features. + /// + /// This method is called upon initialization of the property. + /// + protected virtual IList GetFeatures() => new List + { + PdbFeature.VC140 + }; + + /// + public override uint GetPhysicalSize() + { + return sizeof(uint) // Version + + sizeof(uint) // Signature + + sizeof(uint) // Aage + + 16 // UniqueId + ; + } + + /// + public override void Write(IBinaryStreamWriter writer) + { + throw new NotImplementedException(); + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStreamVersion.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStreamVersion.cs new file mode 100644 index 000000000..361d41e07 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStreamVersion.cs @@ -0,0 +1,19 @@ +namespace AsmResolver.Symbols.Pdb.Metadata.Info; + +/// +/// Provides members defining all possible stream file format versions that PDB defines. +/// +public enum InfoStreamVersion +{ +#pragma warning disable CS1591 + VC2 = 19941610, + VC4 = 19950623, + VC41 = 19950814, + VC50 = 19960307, + VC98 = 19970604, + VC70Dep = 19990604, + VC70 = 20000404, + VC80 = 20030901, + VC110 = 20091201, + VC140 = 20140508, +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/PdbFeature.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/PdbFeature.cs new file mode 100644 index 000000000..16e5d3a66 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/PdbFeature.cs @@ -0,0 +1,28 @@ +namespace AsmResolver.Symbols.Pdb.Metadata.Info; + +/// +/// Provides members defining all possible features that a PDB can have. +/// +public enum PdbFeature : uint +{ + /// + /// Indicates no other feature flags are present, and that an IPI stream is present. + /// + VC110 = 20091201, + + /// + /// Indicates that other feature flags may be present, and that an IPI stream is present. + /// + VC140 = 20140508, + + /// + /// Indicates types can be duplicated in the TPI stream. + /// + NoTypeMerge = 0x4D544F4E, + + /// + /// Indicates the program was linked with /DEBUG:FASTLINK, and all type information is contained in the original + /// object files instead of TPI and IPI streams. + /// + MinimalDebugInfo = 0x494E494D, +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs new file mode 100644 index 000000000..506a9ae86 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using AsmResolver.IO; + +namespace AsmResolver.Symbols.Pdb.Metadata.Info; + +/// +/// Models an PDB info stream by pulling its data from an input stream. +/// +public class SerializedInfoStream : InfoStream +{ + private readonly BinaryStreamReader _reader; + private ulong _featureOffset; + + /// + /// Parses a PDB info stream from an input stream reader. + /// + /// The input stream. + public SerializedInfoStream(BinaryStreamReader reader) + { + Version = (InfoStreamVersion) reader.ReadUInt32(); + Signature = reader.ReadUInt32(); + Age = reader.ReadUInt32(); + + byte[] guidBytes = new byte[16]; + reader.ReadBytes(guidBytes, 0, guidBytes.Length); + + UniqueId = new Guid(guidBytes); + + _reader = reader; + } + + /// + protected override IDictionary GetStreamIndices() + { + var reader = _reader.Fork(); + uint length = reader.ReadUInt32(); + + var stringsReader = reader.ForkRelative(reader.RelativeOffset, length); + var hashTableReader = reader.ForkRelative(reader.RelativeOffset + length); + + var result = PdbHashTable.FromReader(ref hashTableReader, (key, value) => + { + var stringReader = stringsReader.ForkRelative(key); + byte[] rawData = stringReader.ReadBytesUntil(0); + + Utf8String keyString; + if (rawData.Length == 0) + { + keyString = Utf8String.Empty; + } + else + { + // Trim off null terminator byte if its present. + int actualLength = rawData.Length; + if (rawData[actualLength - 1] == 0) + actualLength--; + + keyString = new Utf8String(rawData, 0, actualLength); + } + + return (keyString, (int) value); + }); + + _featureOffset = hashTableReader.Offset; + return result; + } + + /// + protected override IList GetFeatures() + { + // We need to read the stream name->index mapping to be able to read the features list of the PDB. + _ = StreamIndices; + + var result = new List(); + + var reader = _reader.ForkAbsolute(_featureOffset); + while (reader.CanRead(sizeof(uint))) + result.Add((PdbFeature) reader.ReadUInt32()); + + return result; + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs b/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs new file mode 100644 index 000000000..942b1ddd1 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using AsmResolver.IO; + +namespace AsmResolver.Symbols.Pdb.Metadata; + +internal static class PdbHashTable +{ + public static Dictionary FromReader( + ref BinaryStreamReader reader, + Func mapper) + where TKey : notnull + { + uint size = reader.ReadUInt32(); + uint capacity = reader.ReadUInt32(); + + uint presentWordCount = reader.ReadUInt32(); + reader.RelativeOffset += presentWordCount * sizeof(uint); + + uint deletedWordCount = reader.ReadUInt32(); + reader.RelativeOffset += deletedWordCount * sizeof(uint); + + var result = new Dictionary(); + for (int i = 0; i < size; i++) + { + (uint rawKey, uint rawValue) = (reader.ReadUInt32(), reader.ReadUInt32()); + var (key, value) = mapper(rawKey, rawValue); + result.Add(key, value); + } + + uint lastNi = reader.ReadUInt32(); + + return result; + } +} diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs new file mode 100644 index 000000000..97e8827cf --- /dev/null +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using AsmResolver.Symbols.Pdb.Metadata.Info; +using AsmResolver.Symbols.Pdb.Msf; +using Xunit; + +namespace AsmResolver.Symbols.Pdb.Tests.Metadata.Info; + +public class InfoStreamTest +{ + [Fact] + public void Read() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var infoStream = InfoStream.FromReader(file.Streams[1].CreateReader()); + + Assert.Equal(InfoStreamVersion.VC70, infoStream.Version); + Assert.Equal(1u, infoStream.Age); + Assert.Equal(Guid.Parse("205dc366-d8f8-4175-8e06-26dd76722df5"), infoStream.UniqueId); + Assert.Equal(new Dictionary + { + ["/UDTSRCLINEUNDONE"] = 48, + ["/src/headerblock"] = 46, + ["/LinkInfo"] = 5, + ["/TMCache"] = 6, + ["/names"] = 12 + }, infoStream.StreamIndices); + Assert.Equal(new[] {PdbFeature.VC140}, infoStream.Features); + } +} From 8e4de816830345222616fe5da8883c8c7799d74c Mon Sep 17 00:00:00 2001 From: Washi Date: Sat, 18 Jun 2022 14:31:49 +0200 Subject: [PATCH 02/16] Add write support for pdb hash tables and info stream. --- .../Metadata/Info/InfoStream.cs | 39 +++++- .../Metadata/Info/SerializedInfoStream.cs | 2 + .../Metadata/PdbHash.cs | 49 +++++++ .../Metadata/PdbHashTable.cs | 123 +++++++++++++++++- .../Metadata/Info/InfoStreamTest.cs | 14 +- .../Metadata/PdbHashTest.cs | 18 +++ 6 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/PdbHash.cs create mode 100644 test/AsmResolver.Symbols.Pdb.Tests/Metadata/PdbHashTest.cs diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs index 182981fe1..7d3e07c55 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Threading; using AsmResolver.IO; @@ -119,6 +120,42 @@ public override uint GetPhysicalSize() /// public override void Write(IBinaryStreamWriter writer) { - throw new NotImplementedException(); + // Write basic info stream header. + writer.WriteUInt32((uint) Version); + writer.WriteUInt32(Signature); + writer.WriteUInt32(Age); + writer.WriteBytes(UniqueId.ToByteArray()); + + // Construct name buffer, keeping track of the offsets of every name. + using var nameBuffer = new MemoryStream(); + var nameWriter = new BinaryStreamWriter(nameBuffer); + + var stringOffsets = new Dictionary(); + foreach (var entry in StreamIndices) + { + uint offset = (uint) nameWriter.Offset; + nameWriter.WriteBytes(entry.Key.GetBytesUnsafe()); + nameWriter.WriteByte(0); + stringOffsets.Add(entry.Key, offset); + } + + writer.WriteUInt32((uint) nameBuffer.Length); + writer.WriteBytes(nameBuffer.ToArray()); + + // Write the hash table. + // Note: The hash of a single entry is **deliberately** truncated to a 16 bit number. This is because + // the reference implementation of the name table returns a number of type HASH, which is a typedef + // for "unsigned short". If we don't do this, this will result in wrong buckets being filled in the + // hash table, and thus the serialization would fail. See NMTNI::hash() in Microsoft/microsoft-pdb. + StreamIndices.WriteAsPdbHashTable(writer, + str => (ushort) PdbHash.ComputeV1(str), + (key, value) => (stringOffsets[key], (uint) value)); + + // last NI, safe to put always zero. + writer.WriteUInt32(0); + + // Write feature codes. + foreach (var feature in Features) + writer.WriteUInt32((uint) feature); } } diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs index 506a9ae86..53f774848 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs @@ -62,6 +62,8 @@ protected override IDictionary GetStreamIndices() return (keyString, (int) value); }); + uint lastNi = hashTableReader.ReadUInt32(); // Unused. + _featureOffset = hashTableReader.Offset; return result; } diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/PdbHash.cs b/src/AsmResolver.Symbols.Pdb/Metadata/PdbHash.cs new file mode 100644 index 000000000..ed5a6a03b --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/PdbHash.cs @@ -0,0 +1,49 @@ +namespace AsmResolver.Symbols.Pdb.Metadata; + +/// +/// Provides methods for computing hash codes for a PDB hash table. +/// +public static class PdbHash +{ + /// + /// Computes the V1 hash code for a UTF-8 string. + /// + /// The string to compute the hash for. + /// The hash code. + /// + /// See PDB/include/misc.h for reference implementation. + /// + public static unsafe uint ComputeV1(Utf8String value) + { + uint result = 0; + + uint count = (uint) value.ByteCount; + + fixed (byte* ptr = value.GetBytesUnsafe()) + { + byte* p = ptr; + + while (count >= 4) + { + result ^= *(uint*) p; + count -= 4; + p += 4; + } + + if (count >= 2) + { + result ^= *(ushort*) p; + count -= 2; + p += 2; + } + + if (count == 1) + result ^= *p; + } + + result |= 0x20202020; + result ^= result >> 11; + + return result ^ (result >> 16); + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs b/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs index 942b1ddd1..7125fed86 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs @@ -5,8 +5,22 @@ namespace AsmResolver.Symbols.Pdb.Metadata; -internal static class PdbHashTable +/// +/// Provides methods for serializing and deserializing dictionaries as PDB hash tables. +/// +public static class PdbHashTable { + // Reference implementation from PDB/include/map.h + // Specifically, Map::load, Map::find and Map::save. + + /// + /// Reads a single PDB hash table from the input stream and converts it into a dictionary. + /// + /// The input stream to read from. + /// A function that maps the raw key-value pairs into high level constructs. + /// The type of keys in the final dictionary. + /// The type of values in the final dictionary. + /// The reconstructed dictionary. public static Dictionary FromReader( ref BinaryStreamReader reader, Func mapper) @@ -29,8 +43,111 @@ public static Dictionary FromReader( result.Add(key, value); } - uint lastNi = reader.ReadUInt32(); - return result; } + + /// + /// Serializes a dictionary to a PDB hash table. + /// + /// The dictionary to serialize. + /// The output stream to write to. + /// A function that computes the hash code for a single key within the dictionary. + /// A function that maps every key-value pair to raw key-value uint32 pairs. + /// The type of keys in the input dictionary. + /// The type of values in the input dictionary. + public static void WriteAsPdbHashTable( + this IDictionary dictionary, + IBinaryStreamWriter writer, + Func hasher, + Func mapper) + where TKey : notnull + { + var hashTable = dictionary.ToPdbHashTable(hasher, mapper); + + // Write count and capacity. + writer.WriteInt32(dictionary.Count); + writer.WriteUInt32(hashTable.Capacity); + + // Determine which words in the present bitvector to write. + uint wordCount = (hashTable.Capacity + sizeof(uint) - 1) / sizeof(uint); + uint[] words = new uint[wordCount]; + hashTable.Present.CopyTo(words, 0); + while (wordCount > 0 && words[wordCount - 1] == 0) + wordCount--; + + // Write the present bitvector. + writer.WriteUInt32(wordCount); + for (int i = 0; i < wordCount; i++) + writer.WriteUInt32(words[i]); + + // Write deleted bitvector. We just always do 0 (i.e. no deleted buckets). + writer.WriteUInt32(0); + + // Write all buckets. + for (int i = 0; i < hashTable.Keys.Length; i++) + { + if (hashTable.Present.Get(i)) + { + writer.WriteUInt32(hashTable.Keys[i]); + writer.WriteUInt32(hashTable.Values[i]); + } + } + } + + private static HashTableInfo ToPdbHashTable( + this IDictionary dictionary, + Func hasher, + Func mapper) + where TKey : notnull + { + // "Simulate" adding all items to the hash table, effectively calculating the capacity of the map. + // TODO: This can probably be calculated with a single formula instead. + uint capacity = 1; + for (int i = 0; i <= dictionary.Count; i++) + { + // Reference implementation allows only 67% of the capacity to be used. + uint maxLoad = capacity * 2 / 3 + 1; + if (i >= maxLoad) + capacity = 2 * maxLoad; + } + + // Define buckets. + uint[] keys = new uint[capacity]; + uint[] values = new uint[capacity]; + var present = new BitArray((int) capacity, false); + + // Fill in buckets. + foreach (var item in dictionary) + { + uint hash = hasher(item.Key); + (uint key, uint value) = mapper(item.Key, item.Value); + + uint index = hash % capacity; + while (present.Get((int) index)) + index = (index + 1) % capacity; + + keys[index] = key; + values[index] = value; + present.Set((int) index, true); + } + + return new HashTableInfo(capacity, keys, values, present); + } + + private readonly struct HashTableInfo + { + public readonly uint Capacity; + public readonly uint[] Keys; + public readonly uint[] Values; + public readonly BitArray Present; + + public HashTableInfo(uint capacity, uint[] keys, uint[] values, BitArray present) + { + Capacity = capacity; + Keys = keys; + Values = values; + Present = present; + } + } + } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs index 97e8827cf..8c348e1f4 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.IO; +using AsmResolver.IO; using AsmResolver.Symbols.Pdb.Metadata.Info; using AsmResolver.Symbols.Pdb.Msf; using Xunit; @@ -8,11 +10,19 @@ namespace AsmResolver.Symbols.Pdb.Tests.Metadata.Info; public class InfoStreamTest { - [Fact] - public void Read() + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReadWrite(bool rebuild) { var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); var infoStream = InfoStream.FromReader(file.Streams[1].CreateReader()); + if (rebuild) + { + using var stream = new MemoryStream(); + infoStream.Write(new BinaryStreamWriter(stream)); + infoStream = InfoStream.FromReader(ByteArrayDataSource.CreateReader(stream.ToArray())); + } Assert.Equal(InfoStreamVersion.VC70, infoStream.Version); Assert.Equal(1u, infoStream.Age); diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/PdbHashTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/PdbHashTest.cs new file mode 100644 index 000000000..5ca3e791f --- /dev/null +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/PdbHashTest.cs @@ -0,0 +1,18 @@ +using AsmResolver.Symbols.Pdb.Metadata; +using Xunit; + +namespace AsmResolver.Symbols.Pdb.Tests.Metadata; + +public class PdbHashTest +{ + [Theory] + [InlineData("/UDTSRCLINEUNDONE", 0x23296bb2)] + [InlineData("/src/headerblock", 0x2b237ecd)] + [InlineData("/LinkInfo", 0x282209ed)] + [InlineData("/TMCache", 0x2621d5e9)] + [InlineData("/names", 0x6d6cfc21)] + public void HashV1(string value, uint expected) + { + Assert.Equal(expected, PdbHash.ComputeV1(value)); + } +} From 8375199c9f2db75ac14cf4f0e54aa7858ab7a9f1 Mon Sep 17 00:00:00 2001 From: Washi Date: Sat, 18 Jun 2022 14:41:44 +0200 Subject: [PATCH 03/16] Add BinaryStreamReader.ReadBytesUntil overload that can strip off the delimeter. --- .../Strings/SerializedStringsStream.cs | 18 ++------ .../Metadata/Info/SerializedInfoStream.cs | 21 +++------ src/AsmResolver/IO/BinaryStreamReader.cs | 43 +++++++++++++------ 3 files changed, 39 insertions(+), 43 deletions(-) diff --git a/src/AsmResolver.PE/DotNet/Metadata/Strings/SerializedStringsStream.cs b/src/AsmResolver.PE/DotNet/Metadata/Strings/SerializedStringsStream.cs index 55d2f9299..78532dfb1 100644 --- a/src/AsmResolver.PE/DotNet/Metadata/Strings/SerializedStringsStream.cs +++ b/src/AsmResolver.PE/DotNet/Metadata/Strings/SerializedStringsStream.cs @@ -65,21 +65,11 @@ public SerializedStringsStream(string name, in BinaryStreamReader reader) if (!_cachedStrings.TryGetValue(index, out var value) && index < _reader.Length) { var stringsReader = _reader.ForkRelative(index); - byte[] rawData = stringsReader.ReadBytesUntil(0); + byte[] rawData = stringsReader.ReadBytesUntil(0, false); - if (rawData.Length == 0) - { - value = Utf8String.Empty; - } - else - { - // Trim off null terminator byte if its present. - int actualLength = rawData.Length; - if (rawData[actualLength - 1] == 0) - actualLength--; - - value = new Utf8String(rawData, 0, actualLength); - } + value = rawData.Length != 0 + ? new Utf8String(rawData) + : Utf8String.Empty; _cachedStrings[index] = value; } diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs index 53f774848..2cec0ebba 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs @@ -42,22 +42,11 @@ protected override IDictionary GetStreamIndices() var result = PdbHashTable.FromReader(ref hashTableReader, (key, value) => { var stringReader = stringsReader.ForkRelative(key); - byte[] rawData = stringReader.ReadBytesUntil(0); - - Utf8String keyString; - if (rawData.Length == 0) - { - keyString = Utf8String.Empty; - } - else - { - // Trim off null terminator byte if its present. - int actualLength = rawData.Length; - if (rawData[actualLength - 1] == 0) - actualLength--; - - keyString = new Utf8String(rawData, 0, actualLength); - } + byte[] rawData = stringReader.ReadBytesUntil(0, false); + + var keyString = rawData.Length != 0 + ? new Utf8String(rawData) + : Utf8String.Empty; return (keyString, (int) value); }); diff --git a/src/AsmResolver/IO/BinaryStreamReader.cs b/src/AsmResolver/IO/BinaryStreamReader.cs index 74088ce4b..7a48e29be 100644 --- a/src/AsmResolver/IO/BinaryStreamReader.cs +++ b/src/AsmResolver/IO/BinaryStreamReader.cs @@ -328,18 +328,45 @@ public byte[] ReadToEnd() /// /// The delimeter byte to stop at. /// The read bytes, including the delimeter if it was found. - public byte[] ReadBytesUntil(byte delimeter) - { + public byte[] ReadBytesUntil(byte delimeter) => ReadBytesUntil(delimeter, true); + + /// + /// Reads bytes from the input stream until the provided delimeter byte is reached. + /// + /// The delimeter byte to stop at. + /// + /// true if the final delimeter should be included in the return value, false otherwise. + /// + /// The read bytes. + /// + /// This function always consumes the delimeter from the input stream if it is present, regardless of the value + /// of . + /// + public byte[] ReadBytesUntil(byte delimeter, bool includeDelimeterInReturn) + { + bool shouldReadExtra = false; + var lookahead = Fork(); while (lookahead.RelativeOffset < lookahead.Length) { byte b = lookahead.ReadByte(); if (b == delimeter) + { + if (!includeDelimeterInReturn) + { + lookahead.RelativeOffset--; + shouldReadExtra = true; + } break; + } } byte[] buffer = new byte[lookahead.RelativeOffset - RelativeOffset]; ReadBytes(buffer, 0, buffer.Length); + + if (shouldReadExtra) + ReadByte(); + return buffer; } @@ -347,17 +374,7 @@ public byte[] ReadBytesUntil(byte delimeter) /// Reads a null-terminated ASCII string from the input stream. /// /// The read ASCII string, excluding the null terminator. - public string ReadAsciiString() - { - byte[] data = ReadBytesUntil(0); - int length = data.Length; - - // Exclude trailing 0 byte. - if (data[data.Length - 1] == 0) - length--; - - return Encoding.ASCII.GetString(data, 0, length); - } + public string ReadAsciiString() => Encoding.ASCII.GetString(ReadBytesUntil(0, false)); /// /// Reads a zero-terminated Unicode string from the stream. From bc5ce143fdaa8551ac8c60dec2afdfb252c1c9f2 Mon Sep 17 00:00:00 2001 From: Washi Date: Sat, 18 Jun 2022 15:37:22 +0200 Subject: [PATCH 04/16] Add basic read support for DBI stream header. --- .../AsmResolver.Symbols.Pdb.csproj | 1 + .../Metadata/Dbi/DbiAttributes.cs | 30 ++++ .../Metadata/Dbi/DbiStream.cs | 148 ++++++++++++++++++ .../Metadata/Dbi/DbiStreamVersion.cs | 15 ++ .../Metadata/Dbi/SerializedDbiStream.cs | 51 ++++++ .../Metadata/Info/InfoStream.cs | 9 +- .../Metadata/Info/InfoStreamVersion.cs | 1 + .../Metadata/Info/SerializedInfoStream.cs | 2 +- .../Metadata/Dbi/DbiStreamTest.cs | 20 +++ .../Metadata/Info/InfoStreamTest.cs | 2 +- 10 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiAttributes.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStreamVersion.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs create mode 100644 test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs diff --git a/src/AsmResolver.Symbols.Pdb/AsmResolver.Symbols.Pdb.csproj b/src/AsmResolver.Symbols.Pdb/AsmResolver.Symbols.Pdb.csproj index b0b3df765..168f407cc 100644 --- a/src/AsmResolver.Symbols.Pdb/AsmResolver.Symbols.Pdb.csproj +++ b/src/AsmResolver.Symbols.Pdb/AsmResolver.Symbols.Pdb.csproj @@ -21,6 +21,7 @@ + diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiAttributes.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiAttributes.cs new file mode 100644 index 000000000..3fb37697a --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiAttributes.cs @@ -0,0 +1,30 @@ +using System; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Provides members defining all attributes that can be assigned to a single DBI stream. +/// +[Flags] +public enum DbiAttributes : ushort +{ + /// + /// Indicates no attributes were assigned. + /// + None = 0, + + /// + /// Indicates the program was linked in an incremental manner. + /// + IncrementallyLinked = 1, + + /// + /// Indicates private symbols were stripped from the PDB file. + /// + PrivateSymbolsStripped = 2, + + /// + /// Indicates the program was linked using link.exe with the undocumented /DEBUG:CTYPES flag. + /// + HasConflictingTypes = 4, +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs new file mode 100644 index 000000000..f615a3545 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -0,0 +1,148 @@ +using AsmResolver.IO; +using AsmResolver.PE.File.Headers; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Represents the DBI Stream (also known as the Debug Information stream). +/// +public class DbiStream : SegmentBase +{ + /// + /// Gets the default fixed MSF stream index for the DBI stream. + /// + public const int StreamIndex = 3; + + /// + /// Gets or sets the version signature assigned to the DBI stream. + /// + /// + /// This value should always be -1 for valid PDB files. + /// + public int VersionSignature + { + get; + set; + } = -1; + + /// + /// Gets or sets the version number of the DBI header. + /// + /// + /// Modern tooling only recognize the VC7.0 file format. + /// + public DbiStreamVersion VersionHeader + { + get; + set; + } = DbiStreamVersion.V70; + + /// + /// Gets or sets the number of times the DBI stream has been written. + /// + public uint Age + { + get; + set; + } = 1; + + /// + /// Gets or sets the MSF stream index of the Global Symbol Stream. + /// + public ushort GlobalStreamIndex + { + get; + set; + } + + /// + /// Gets or sets a bitfield containing the major and minor version of the toolchain that was used to build the program. + /// + public ushort BuildNumber + { + get; + set; + } + + /// + /// Gets or sets the MSF stream index of the Public Symbol Stream. + /// + public ushort PublicStreamIndex + { + get; + set; + } + + /// + /// Gets or sets the version number of mspdbXXXX.dll that was used to produce this PDB file. + /// + public ushort PdbDllVersion + { + get; + set; + } + + /// + /// Gets or sets the MSF stream index of the Symbol Record Stream. + /// + public ushort SymbolRecordStreamIndex + { + get; + set; + } + + /// + /// Unknown. + /// + public ushort PdbDllRbld + { + get; + set; + } + + /// + /// Gets or sets the MSF stream index of the MFC type server. + /// + public uint MfcTypeServerIndex + { + get; + set; + } + + /// + /// Gets or sets attributes associated to the DBI stream. + /// + public DbiAttributes Attributes + { + get; + set; + } + + /// + /// Gets or sets the machine type the program was compiled for. + /// + public MachineType Machine + { + get; + set; + } + + /// + /// Reads a single DBI stream from the provided input stream. + /// + /// The input stream. + /// The parsed DBI stream. + public static DbiStream FromReader(BinaryStreamReader reader) => new SerializedDbiStream(reader); + + /// + public override uint GetPhysicalSize() + { + throw new System.NotImplementedException(); + } + + /// + public override void Write(IBinaryStreamWriter writer) + { + throw new System.NotImplementedException(); + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStreamVersion.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStreamVersion.cs new file mode 100644 index 000000000..fe76de860 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStreamVersion.cs @@ -0,0 +1,15 @@ +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Provides members defining all possible DBI stream format version numbers. +/// +public enum DbiStreamVersion +{ +#pragma warning disable CS1591 + VC41 = 930803, + V50 = 19960307, + V60 = 19970606, + V70 = 19990903, + V110 = 20091201 +#pragma warning restore CS1591 +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs new file mode 100644 index 000000000..5bb1043d5 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -0,0 +1,51 @@ +using AsmResolver.IO; +using AsmResolver.PE.File.Headers; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Implements a DBI stream that pulls its data from an input stream. +/// +public class SerializedDbiStream : DbiStream +{ + private readonly uint _moduleInfoSize; + private readonly uint _sectionContributionSize; + private readonly uint _sectionMapSize; + private readonly uint _sourceInfoSize; + private readonly uint _typeServerMapSize; + private readonly uint _optionalDebugHeaderSize; + private readonly uint _ecSize; + + /// + /// Parses a DBI stream from an input stream reader. + /// + /// The input stream. + public SerializedDbiStream(BinaryStreamReader reader) + { + VersionSignature = reader.ReadInt32(); + VersionHeader = (DbiStreamVersion) reader.ReadUInt32(); + Age = reader.ReadUInt32(); + GlobalStreamIndex = reader.ReadUInt16(); + BuildNumber = reader.ReadUInt16(); + PublicStreamIndex = reader.ReadUInt16(); + PdbDllVersion = reader.ReadUInt16(); + SymbolRecordStreamIndex = reader.ReadUInt16(); + PdbDllRbld = reader.ReadUInt16(); + + _moduleInfoSize = reader.ReadUInt32(); + _sectionContributionSize = reader.ReadUInt32(); + _sectionMapSize = reader.ReadUInt32(); + _sourceInfoSize = reader.ReadUInt32(); + _typeServerMapSize = reader.ReadUInt32(); + + MfcTypeServerIndex = reader.ReadUInt32(); + + _optionalDebugHeaderSize = reader.ReadUInt32(); + _ecSize = reader.ReadUInt32(); + + Attributes = (DbiAttributes) reader.ReadUInt16(); + Machine = (MachineType) reader.ReadUInt16(); + + _ = reader.ReadUInt32(); + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs index 7d3e07c55..00fede54c 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs @@ -11,6 +11,11 @@ namespace AsmResolver.Symbols.Pdb.Metadata.Info; /// public class InfoStream : SegmentBase { + /// + /// Gets the default fixed MSF stream index for the PDB Info stream. + /// + public const int StreamIndex = 1; + private IDictionary? _streamIndices; private IList? _features; @@ -24,7 +29,7 @@ public InfoStreamVersion Version { get; set; - } + } = InfoStreamVersion.VC70; /// /// Gets or sets the 32-bit UNIX time-stamp of the PDB file. @@ -42,7 +47,7 @@ public uint Age { get; set; - } + } = 1; /// /// Gets or sets the unique identifier assigned to the PDB file. diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStreamVersion.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStreamVersion.cs index 361d41e07..d377f5ba1 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStreamVersion.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStreamVersion.cs @@ -16,4 +16,5 @@ public enum InfoStreamVersion VC80 = 20030901, VC110 = 20091201, VC140 = 20140508, +#pragma warning restore CS1591 } diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs index 2cec0ebba..8ce5c3b21 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs @@ -5,7 +5,7 @@ namespace AsmResolver.Symbols.Pdb.Metadata.Info; /// -/// Models an PDB info stream by pulling its data from an input stream. +/// Implements an PDB info stream that pulls its data from an input stream. /// public class SerializedInfoStream : InfoStream { diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs new file mode 100644 index 000000000..e1bca57a4 --- /dev/null +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -0,0 +1,20 @@ +using AsmResolver.PE.File.Headers; +using AsmResolver.Symbols.Pdb.Metadata.Dbi; +using AsmResolver.Symbols.Pdb.Msf; +using Xunit; + +namespace AsmResolver.Symbols.Pdb.Tests.Metadata.Dbi; + +public class DbiStreamTest +{ + [Fact] + public void Read() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); + + Assert.Equal(1u, dbiStream.Age); + Assert.Equal(DbiAttributes.None, dbiStream.Attributes); + Assert.Equal(MachineType.I386, dbiStream.Machine); + } +} diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs index 8c348e1f4..ff78a7b81 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs @@ -16,7 +16,7 @@ public class InfoStreamTest public void ReadWrite(bool rebuild) { var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); - var infoStream = InfoStream.FromReader(file.Streams[1].CreateReader()); + var infoStream = InfoStream.FromReader(file.Streams[InfoStream.StreamIndex].CreateReader()); if (rebuild) { using var stream = new MemoryStream(); From 3ef199a1c9241dd210d0f1fa6b6cc6ab7212ecc9 Mon Sep 17 00:00:00 2001 From: Washi Date: Sat, 18 Jun 2022 16:56:30 +0200 Subject: [PATCH 05/16] Add read support for dbi module descriptors. --- .../Metadata/Dbi/DbiStream.cs | 23 ++ .../Metadata/Dbi/ModuleDescriptor.cs | 207 ++++++++++++++++++ .../Dbi/ModuleDescriptorAttributes.cs | 25 +++ .../Metadata/Dbi/SectionContribution.cs | 124 +++++++++++ .../Metadata/Dbi/SerializedDbiStream.cs | 56 +++-- .../Metadata/Dbi/DbiStreamTest.cs | 46 +++- .../Msf/MsfFileTest.cs | 1 + 7 files changed, 467 insertions(+), 15 deletions(-) create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptorAttributes.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index f615a3545..feb56f542 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; +using System.Threading; using AsmResolver.IO; using AsmResolver.PE.File.Headers; @@ -13,6 +15,8 @@ public class DbiStream : SegmentBase /// public const int StreamIndex = 3; + private IList? _modules; + /// /// Gets or sets the version signature assigned to the DBI stream. /// @@ -127,6 +131,25 @@ public MachineType Machine set; } + /// + /// Gets a collection of modules (object files) that were linked together into the program. + /// + public IList Modules + { + get + { + if (_modules is null) + Interlocked.CompareExchange(ref _modules, GetModules(), null); + return _modules; + } + } + + /// + /// Obtains the list of modules + /// + /// + protected virtual IList GetModules() => new List(); + /// /// Reads a single DBI stream from the provided input stream. /// diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs new file mode 100644 index 000000000..968777a1d --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs @@ -0,0 +1,207 @@ +using AsmResolver.IO; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Represents a reference to a single module (object file) that was linked into a program. +/// +public class ModuleDescriptor : IWritable +{ + /// + /// Gets or sets a description of the section within the final binary which contains code + /// and/or data from this module. + /// + public SectionContribution SectionContribution + { + get; + set; + } = new(); + + /// + /// Gets or sets the attributes assigned to this module descriptor. + /// + public ModuleDescriptorAttributes Attributes + { + get; + set; + } + + /// + /// Gets or sets the index of the type server for this module. + /// + public ushort TypeServerIndex + { + get => (ushort) ((ushort) Attributes >> 8); + set => Attributes = (Attributes & ~ModuleDescriptorAttributes.TsmMask) | (ModuleDescriptorAttributes) (value << 8); + } + + /// + /// Gets or sets the MSF stream index of the stream that the symbols of this module. + /// + public ushort SymbolStreamIndex + { + get; + set; + } + + /// + /// Gets or sets the size of the CodeView data within the module's symbol stream. + /// + public uint SymbolDataSize + { + get; + set; + } + + /// + /// Gets or sets the size of the C11-style CodeView data within the module's symbol stream. + /// + public uint SymbolC11DataSize + { + get; + set; + } + + /// + /// Gets or sets the size of the C13-style CodeView data within the module's symbol stream. + /// + public uint SymbolC13DataSize + { + get; + set; + } + + /// + /// Gets or sets the number of source files that contributed to this module during the compilation. + /// + public ushort SourceFileCount + { + get; + set; + } + + /// + /// Gets or sets the offset in the names buffer of the primary translation unit. + /// + /// + /// For most compilers this value is set to zero. + /// + public uint SourceFileNameIndex + { + get; + set; + } + + /// + /// Gets or sets the offset in the names buffer of the PDB file. + /// + /// + /// For most modules (except the special * LINKER * module) this value is set to zero. + /// + public uint PdbFilePathNameIndex + { + get; + set; + } + + /// + /// Gets or sets the name of the module. + /// + /// + /// This is often a full path to the object file that was passed into link.exe directly, or a string in the + /// form of Import:dll_name + /// + public Utf8String? ModuleName + { + get; + set; + } + + /// + /// Gets or sets the name of the object file name. + /// + /// + /// In the case this module is linked directly passed to link.exe, this is the same as . + /// If the module comes from an archive, this is the full path to that archive. + /// + public Utf8String? ObjectFileName + { + get; + set; + } + + /// + /// Parses a single module descriptor from the provided input stream. + /// + /// The input stream. + /// THe parsed module descriptor. + public static ModuleDescriptor FromReader(ref BinaryStreamReader reader) + { + var result = new ModuleDescriptor(); + + reader.ReadUInt32(); + result.SectionContribution = SectionContribution.FromReader(ref reader); + result.Attributes = (ModuleDescriptorAttributes) reader.ReadUInt16(); + result.SymbolStreamIndex = reader.ReadUInt16(); + result.SymbolDataSize = reader.ReadUInt32(); + result.SymbolC11DataSize = reader.ReadUInt32(); + result.SymbolC13DataSize = reader.ReadUInt32(); + result.SourceFileCount = reader.ReadUInt16(); + reader.ReadUInt16(); + reader.ReadUInt32(); + result.SourceFileNameIndex = reader.ReadUInt32(); + result.PdbFilePathNameIndex = reader.ReadUInt32(); + result.ModuleName = new Utf8String(reader.ReadBytesUntil(0, false)); + result.ObjectFileName = new Utf8String(reader.ReadBytesUntil(0, false)); + reader.Align(4); + + return result; + } + + /// + public uint GetPhysicalSize() + { + return sizeof(uint) // Unused1 + + SectionContribution.GetPhysicalSize() // SectionContribution + + sizeof(ModuleDescriptorAttributes) // Attributes + + sizeof(ushort) // SymbolStreamIndex + + sizeof(uint) // SymbolDataSize + + sizeof(uint) // SymbolC11DataSize + + sizeof(uint) // SymbolC13DataSize + + sizeof(ushort) // SourceFileCount + + sizeof(char) * 2 // Padding + + sizeof(uint) // Unused2 + + sizeof(uint) // SourceFileNameIndex + + sizeof(uint) // PdbFilePathNameIndex + + (uint) (ModuleName?.ByteCount ?? 0) + 1 // ModuleName + + (uint) (ObjectFileName?.ByteCount ?? 0) + 1 // ObjectFileName + ; + } + + /// + public void Write(IBinaryStreamWriter writer) + { + writer.WriteUInt32(0); + SectionContribution.Write(writer); + writer.WriteUInt16((ushort) Attributes); + writer.WriteUInt16(SymbolStreamIndex); + writer.WriteUInt32(SymbolDataSize); + writer.WriteUInt32(SymbolC11DataSize); + writer.WriteUInt32(SymbolC13DataSize); + writer.WriteUInt16(SourceFileCount); + writer.WriteUInt16(0); + writer.WriteUInt32(0); + writer.WriteUInt32(SourceFileNameIndex); + writer.WriteUInt32(PdbFilePathNameIndex); + if (ModuleName is not null) + writer.WriteBytes(ModuleName.GetBytesUnsafe()); + writer.WriteByte(0); + if (ObjectFileName is not null) + writer.WriteBytes(ObjectFileName.GetBytesUnsafe()); + writer.WriteByte(0); + writer.Align(4); + } + + /// + public override string ToString() => ModuleName ?? ObjectFileName ?? "?"; +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptorAttributes.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptorAttributes.cs new file mode 100644 index 000000000..713f251b9 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptorAttributes.cs @@ -0,0 +1,25 @@ +using System; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Defines all possible flags that can be assigned to a module descriptor. +/// +[Flags] +public enum ModuleDescriptorAttributes : ushort +{ + /// + /// Indicates the module has been written to since reading the PDB. + /// + Dirty = 1, + + /// + /// Indicates the module contains Edit & Continue information. + /// + EC = 2, + + /// + /// Provides a mask for the type server index that is stored within the flags. + /// + TsmMask = 0xFF00, +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs new file mode 100644 index 000000000..00b684dd5 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs @@ -0,0 +1,124 @@ +using AsmResolver.IO; +using AsmResolver.PE.File.Headers; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Describes the section in the final executable file that a particular object or module is stored at. +/// +public class SectionContribution : IWritable +{ + /// + /// Gets or sets the index of the section. + /// + public ushort Section + { + get; + set; + } + + /// + /// Gets or sets the offset within the section that this contribution starts at. + /// + public uint Offset + { + get; + set; + } + + /// + /// Gets or sets the size of the section contribution. + /// + public uint Size + { + get; + set; + } + + /// + /// Gets or sets the section flags and permissions associated to this section contribution. + /// + public SectionFlags Characteristics + { + get; + set; + } + + /// + /// Gets or sets the index of the module. + /// + public ushort ModuleIndex + { + get; + set; + } + + /// + /// Gets or sets a cyclic redundancy code that can be used to verify the data section of this contribution. + /// + public uint DataCrc + { + get; + set; + } + + /// + /// Gets or sets a cyclic redundancy code that can be used to verify the relocation section of this contribution. + /// + public uint RelocCrc + { + get; + set; + } + + /// + /// Parses a single section contribution from the provided input stream. + /// + /// The input stream. + /// The parsed section contribution. + public static SectionContribution FromReader(ref BinaryStreamReader reader) + { + var result = new SectionContribution(); + + result.Section = reader.ReadUInt16(); + reader.ReadUInt16(); + result.Offset = reader.ReadUInt32(); + result.Size = reader.ReadUInt32(); + result.Characteristics = (SectionFlags) reader.ReadUInt32(); + result.ModuleIndex = reader.ReadUInt16(); + reader.ReadUInt16(); + result.DataCrc = reader.ReadUInt32(); + result.RelocCrc = reader.ReadUInt32(); + + return result; + } + + /// + public uint GetPhysicalSize() + { + return sizeof(ushort) // Section + + sizeof(ushort) // Padding1 + + sizeof(uint) // Offset + + sizeof(uint) // Size + + sizeof(uint) // Characteristics + + sizeof(ushort) // ModuleIndex + + sizeof(ushort) // Padding2 + + sizeof(uint) // DataCrc + + sizeof(uint) // RelocCrc + ; + } + + /// + public void Write(IBinaryStreamWriter writer) + { + writer.WriteUInt16(Section); + writer.WriteUInt16(0); + writer.WriteUInt32(Offset); + writer.WriteUInt32(Size); + writer.WriteUInt32((uint) Characteristics); + writer.WriteUInt16(ModuleIndex); + writer.WriteUInt16(0); + writer.WriteUInt32(DataCrc); + writer.WriteUInt32(RelocCrc); + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index 5bb1043d5..93def65e3 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using AsmResolver.IO; using AsmResolver.PE.File.Headers; @@ -8,13 +9,13 @@ namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; /// public class SerializedDbiStream : DbiStream { - private readonly uint _moduleInfoSize; - private readonly uint _sectionContributionSize; - private readonly uint _sectionMapSize; - private readonly uint _sourceInfoSize; - private readonly uint _typeServerMapSize; - private readonly uint _optionalDebugHeaderSize; - private readonly uint _ecSize; + private readonly BinaryStreamReader _moduleInfoReader; + private readonly BinaryStreamReader _sectionContributionReader; + private readonly BinaryStreamReader _sectionMapReader; + private readonly BinaryStreamReader _sourceInfoReader; + private readonly BinaryStreamReader _typeServerMapReader; + private readonly BinaryStreamReader _optionalDebugHeaderReader; + private readonly BinaryStreamReader _ecReader; /// /// Parses a DBI stream from an input stream reader. @@ -32,20 +33,47 @@ public SerializedDbiStream(BinaryStreamReader reader) SymbolRecordStreamIndex = reader.ReadUInt16(); PdbDllRbld = reader.ReadUInt16(); - _moduleInfoSize = reader.ReadUInt32(); - _sectionContributionSize = reader.ReadUInt32(); - _sectionMapSize = reader.ReadUInt32(); - _sourceInfoSize = reader.ReadUInt32(); - _typeServerMapSize = reader.ReadUInt32(); + uint moduleInfoSize = reader.ReadUInt32(); + uint sectionContributionSize = reader.ReadUInt32(); + uint sectionMapSize = reader.ReadUInt32(); + uint sourceInfoSize = reader.ReadUInt32(); + uint typeServerMapSize = reader.ReadUInt32(); MfcTypeServerIndex = reader.ReadUInt32(); - _optionalDebugHeaderSize = reader.ReadUInt32(); - _ecSize = reader.ReadUInt32(); + uint optionalDebugHeaderSize = reader.ReadUInt32(); + uint ecSize = reader.ReadUInt32(); Attributes = (DbiAttributes) reader.ReadUInt16(); Machine = (MachineType) reader.ReadUInt16(); _ = reader.ReadUInt32(); + + _moduleInfoReader = reader.ForkRelative(reader.RelativeOffset, moduleInfoSize); + reader.Offset += moduleInfoSize; + _sectionContributionReader = reader.ForkRelative(reader.RelativeOffset, sectionContributionSize); + reader.Offset += sectionContributionSize; + _sectionMapReader = reader.ForkRelative(reader.RelativeOffset, sectionMapSize); + reader.Offset += sectionMapSize; + _sourceInfoReader = reader.ForkRelative(reader.RelativeOffset, sourceInfoSize); + reader.Offset += sourceInfoSize; + _typeServerMapReader = reader.ForkRelative(reader.RelativeOffset, typeServerMapSize); + reader.Offset += typeServerMapSize; + _ecReader = reader.ForkRelative(reader.RelativeOffset, ecSize); + reader.Offset += ecSize; + _optionalDebugHeaderReader = reader.ForkRelative(reader.RelativeOffset, optionalDebugHeaderSize); + reader.Offset += optionalDebugHeaderSize; + } + + /// + protected override IList GetModules() + { + var result = new List(); + + var reader = _moduleInfoReader.Fork(); + while (reader.CanRead(1)) + result.Add(ModuleDescriptor.FromReader(ref reader)); + + return result; } } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs index e1bca57a4..d5fe61bee 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -1,3 +1,4 @@ +using System.Linq; using AsmResolver.PE.File.Headers; using AsmResolver.Symbols.Pdb.Metadata.Dbi; using AsmResolver.Symbols.Pdb.Msf; @@ -8,7 +9,7 @@ namespace AsmResolver.Symbols.Pdb.Tests.Metadata.Dbi; public class DbiStreamTest { [Fact] - public void Read() + public void ReadHeader() { var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); @@ -17,4 +18,47 @@ public void Read() Assert.Equal(DbiAttributes.None, dbiStream.Attributes); Assert.Equal(MachineType.I386, dbiStream.Machine); } + + [Fact] + public void ReadModuleNames() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); + + Assert.Equal(new[] + { + "* CIL *", + "C:\\Users\\Admin\\source\\repos\\AsmResolver\\test\\TestBinaries\\Native\\SimpleDll\\Release\\dllmain.obj", + "C:\\Users\\Admin\\source\\repos\\AsmResolver\\test\\TestBinaries\\Native\\SimpleDll\\Release\\pch.obj", + "* Linker Generated Manifest RES *", + "Import:KERNEL32.dll", + "KERNEL32.dll", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\sehprolg4.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_cookie.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_report.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_support.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\guard_support.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\loadcfg.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\dyn_tls_init.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\ucrt_detection.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\cpu_disp.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\chandler4gs.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\secchk.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\argv_mode.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\default_local_stdio_options.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\tncleanup.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\dll_dllmain.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\initializers.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\utility.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\ucrt_stubs.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\utility_desktop.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\initsect.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\x86_exception_filter.obj", + "VCRUNTIME140.dll", + "Import:VCRUNTIME140.dll", + "Import:api-ms-win-crt-runtime-l1-1-0.dll", + "api-ms-win-crt-runtime-l1-1-0.dll", + "* Linker *", + }, dbiStream.Modules.Select(m => m.ModuleName?.Value)); + } } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Msf/MsfFileTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Msf/MsfFileTest.cs index b17cd4b36..e7c589c70 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Msf/MsfFileTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Msf/MsfFileTest.cs @@ -1,5 +1,6 @@ using System.IO; using System.Linq; +using AsmResolver.IO; using AsmResolver.Symbols.Pdb.Msf; using Xunit; From 09d5166860fad57d5db218a228d43e0d4b5caddf Mon Sep 17 00:00:00 2001 From: Washi Date: Sun, 19 Jun 2022 22:19:30 +0200 Subject: [PATCH 06/16] Add read support section contributions to DBI header. --- .../Metadata/Dbi/DbiStream.cs | 30 +++++++++++++++-- .../Metadata/Dbi/SectionContribution.cs | 1 + .../Dbi/SectionContributionStreamVersion.cs | 17 ++++++++++ .../Metadata/Dbi/SerializedDbiStream.cs | 18 +++++++++++ .../Metadata/Dbi/DbiStreamTest.cs | 32 +++++++++++++++++++ 5 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContributionStreamVersion.cs diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index feb56f542..5af75e2fd 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -16,6 +16,7 @@ public class DbiStream : SegmentBase public const int StreamIndex = 3; private IList? _modules; + private IList? _sectionContributions; /// /// Gets or sets the version signature assigned to the DBI stream. @@ -145,11 +146,36 @@ public IList Modules } /// - /// Obtains the list of modules + /// Gets a collection of section contributions describing the layout of the sections of the final executable file. /// - /// + public IList SectionContributions + { + get + { + if (_sectionContributions is null) + Interlocked.CompareExchange(ref _sectionContributions, GetSectionContributions(), null); + return _sectionContributions; + } + } + + /// + /// Obtains the list of module descriptors. + /// + /// The module descriptors + /// + /// This method is called upon initialization of the property. + /// protected virtual IList GetModules() => new List(); + /// + /// Obtains the list of section contributions. + /// + /// The section contributions. + /// + /// This method is called upon initialization of the property. + /// + protected virtual IList GetSectionContributions() => new List(); + /// /// Reads a single DBI stream from the provided input stream. /// diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs index 00b684dd5..35953c208 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs @@ -1,3 +1,4 @@ +using System.Text; using AsmResolver.IO; using AsmResolver.PE.File.Headers; diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContributionStreamVersion.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContributionStreamVersion.cs new file mode 100644 index 000000000..48daaf3a4 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContributionStreamVersion.cs @@ -0,0 +1,17 @@ +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Provides members defining all valid versions of the Section Contribution sub stream's file format. +/// +public enum SectionContributionStreamVersion : uint +{ + /// + /// Indicates version 6.0 is used. + /// + Ver60 = 0xeffe0000 + 19970605, + + /// + /// Indicates version 2.0 is used. + /// + V2 = 0xeffe0000 + 20140516 +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index 93def65e3..dd3c1b42e 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -76,4 +76,22 @@ protected override IList GetModules() return result; } + + /// + protected override IList GetSectionContributions() + { + var result = new List(); + + var reader = _sectionContributionReader.Fork(); + var version = (SectionContributionStreamVersion) reader.ReadUInt32(); + + while (reader.CanRead(1)) + { + result.Add(SectionContribution.FromReader(ref reader)); + if (version == SectionContributionStreamVersion.V2) + reader.ReadUInt32(); + } + + return result; + } } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs index d5fe61bee..280266228 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -61,4 +61,36 @@ public void ReadModuleNames() "* Linker *", }, dbiStream.Modules.Select(m => m.ModuleName?.Value)); } + + [Fact] + public void ReadSectionContributions() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); + + Assert.Equal(new (ushort, uint)[] + { + (1, 1669053862), (16, 2162654757), (20, 1635644926), (20, 3159649454), (20, 1649652954), (20, 3877379438), + (20, 4262788820), (20, 199934614), (8, 4235719287), (8, 1374843914), (9, 4241735292), (9, 2170796787), + (19, 1300950661), (19, 3968158929), (18, 3928463356), (18, 3928463356), (18, 2109213706), (22, 1457516325), + (22, 3939645857), (22, 1393694582), (22, 546064581), (22, 1976627334), (22, 513172946), (22, 25744891), + (22, 1989765812), (22, 2066266302), (22, 3810887196), (22, 206965504), (22, 647717352), (22, 3911072265), + (22, 3290064241), (12, 3928463356), (24, 2717331243), (24, 3687876222), (25, 2318145338), (25, 2318145338), + (6, 542071654), (15, 1810708069), (10, 3974941622), (14, 1150179208), (17, 2709606169), (13, 2361171624), + (28, 0), (28, 0), (28, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), + (23, 3467414241), (23, 4079273803), (26, 1282639619), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), + (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (28, 0), (28, 0), (28, 0), (27, 0), (29, 0), (29, 0), + (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (10, 2556510175), (21, 2556510175), + (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), + (21, 2556510175), (20, 2556510175), (8, 4117779887), (31, 0), (11, 525614319), (31, 0), (31, 0), (31, 0), + (31, 0), (31, 0), (25, 2556510175), (25, 2556510175), (25, 2556510175), (25, 2556510175), (20, 3906165615), + (20, 1185345766), (20, 407658226), (22, 2869884627), (27, 0), (30, 0), (5, 0), (27, 0), (4, 0), (4, 0), + (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (28, 0), (28, 0), (28, 0), + (27, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (28, 0), (28, 0), + (28, 0), (27, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (4, 0), + (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (7, 4096381681), + (22, 454268333), (14, 1927129959), (23, 1927129959), (20, 0), (8, 0), (19, 0), (18, 0), (18, 0), (22, 0), + (24, 0), (10, 0), (14, 0), (2, 0), (31, 0), (3, 0), (3, 0) + }, dbiStream.SectionContributions.Select(x => (x.ModuleIndex, x.DataCrc))); + } } From f9828030787f6a97a8af4a230b4a39b4089a9b62 Mon Sep 17 00:00:00 2001 From: Washi Date: Sun, 19 Jun 2022 23:52:12 +0200 Subject: [PATCH 07/16] Add read support section mappings in DBI stream. --- .../Metadata/Dbi/DbiStream.cs | 28 ++++ .../Dbi/ModuleDescriptorAttributes.cs | 2 +- .../Metadata/Dbi/SectionMap.cs | 128 ++++++++++++++++++ .../Metadata/Dbi/SectionMapAttributes.cs | 42 ++++++ .../Metadata/Dbi/SerializedDbiStream.cs | 16 +++ .../Metadata/Dbi/DbiStreamTest.cs | 20 +++ 6 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMap.cs create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMapAttributes.cs diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index 5af75e2fd..4a0e490fe 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -17,6 +17,7 @@ public class DbiStream : SegmentBase private IList? _modules; private IList? _sectionContributions; + private IList? _sectionMaps; /// /// Gets or sets the version signature assigned to the DBI stream. @@ -158,6 +159,24 @@ public IList SectionContributions } } + /// + /// Gets a collection of section mappings stored in the section mapping sub stream. + /// + /// + /// The exact purpose of this is unknown, but it seems to be always containing a copy of the sections in the final + /// executable file. + /// + public IList SectionMaps + { + get + { + if (_sectionMaps is null) + Interlocked.CompareExchange(ref _sectionMaps, GetSectionMaps(), null); + return _sectionMaps; + + } + } + /// /// Obtains the list of module descriptors. /// @@ -176,6 +195,15 @@ public IList SectionContributions /// protected virtual IList GetSectionContributions() => new List(); + /// + /// Obtains the list of section maps. + /// + /// The section maps. + /// + /// This method is called upon initialization of the property. + /// + protected virtual IList GetSectionMaps() => new List(); + /// /// Reads a single DBI stream from the provided input stream. /// diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptorAttributes.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptorAttributes.cs index 713f251b9..5b6a018c0 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptorAttributes.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptorAttributes.cs @@ -14,7 +14,7 @@ public enum ModuleDescriptorAttributes : ushort Dirty = 1, /// - /// Indicates the module contains Edit & Continue information. + /// Indicates the module contains Edit & Continue information. /// EC = 2, diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMap.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMap.cs new file mode 100644 index 000000000..6c42bb300 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMap.cs @@ -0,0 +1,128 @@ +using AsmResolver.IO; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Represents a single entry in the Section Map sub stream of the DBI stream. +/// +public class SectionMap : IWritable +{ + /// + /// Gets or sets the attributes assigned to this section map. + /// + public SectionMapAttributes Attributes + { + get; + set; + } + + /// + /// Gets or sets the logical overlay number of this section map. + /// + public ushort LogicalOverlayNumber + { + get; + set; + } + + /// + /// Gets or sets the group index into the descriptor array. + /// + public ushort Group + { + get; + set; + } + + /// + /// Gets or sets the frame index. + /// + public ushort Frame + { + get; + set; + } + + /// + /// Gets or sets the byte offset of the segment or group name in string table, or 0xFFFF if no name was assigned. + /// + public ushort SectionName + { + get; + set; + } + + /// + /// Gets or sets the byte offset of the class in the string table, or 0xFFFF if no name was assigned.. + /// + public ushort ClassName + { + get; + set; + } + + /// + /// Gets or sets the byte offset of the logical segment within physical segment. If group is set in flags, this is the offset of the group. + /// + public uint Offset + { + get; + set; + } + + /// + /// Gets or sets the number of bytes that the segment or group consists of. + /// + public uint SectionLength + { + get; + set; + } + + /// + /// Parses a single section map from the provided input stream. + /// + /// The input stream. + /// The parsed section map. + public static SectionMap FromReader(ref BinaryStreamReader reader) + { + return new SectionMap + { + Attributes = (SectionMapAttributes) reader.ReadUInt16(), + LogicalOverlayNumber = reader.ReadUInt16(), + Group = reader.ReadUInt16(), + Frame = reader.ReadUInt16(), + SectionName = reader.ReadUInt16(), + ClassName = reader.ReadUInt16(), + Offset = reader.ReadUInt32(), + SectionLength = reader.ReadUInt32() + }; + } + + /// + public uint GetPhysicalSize() + { + return sizeof(ushort) // Attributes + + sizeof(ushort) // Ovl + + sizeof(ushort) // Group + + sizeof(ushort) // Frame + + sizeof(ushort) // SectionName + + sizeof(ushort) // ClassName + + sizeof(uint) // Offset + + sizeof(uint) // SectionLength + ; + } + + /// + public void Write(IBinaryStreamWriter writer) + { + writer.WriteUInt16((ushort) Attributes); + writer.WriteUInt16(LogicalOverlayNumber); + writer.WriteUInt16(Group); + writer.WriteUInt16(Frame); + writer.WriteUInt16(SectionName); + writer.WriteUInt16(ClassName); + writer.WriteUInt32(Offset); + writer.WriteUInt32(SectionLength); + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMapAttributes.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMapAttributes.cs new file mode 100644 index 000000000..7f6449772 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMapAttributes.cs @@ -0,0 +1,42 @@ +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Provides members describing all possible attributes that can be assigned to a single section map entry. +/// +public enum SectionMapAttributes : ushort +{ + /// + /// Indicates the segment is readable. + /// + Read = 1 << 0, + + /// + /// Indicates the segment is writable. + /// + Write = 1 << 1, + + /// + /// Indicates the segment is executable. + /// + Execute = 1 << 2, + + /// + /// Indicates the descriptor describes a 32-bit linear address. + /// + AddressIs32Bit = 1 << 3, + + /// + /// Indicates the frame represents a selector. + /// + IsSelector = 1 << 8, + + /// + /// Indicates the frame represents an absolute address. + /// + IsAbsoluteAddress = 1 << 9, + + /// + /// Indicates the descriptor represents a group. + /// + IsGroup = 1 << 10 +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index dd3c1b42e..6e0219b57 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -94,4 +94,20 @@ protected override IList GetSectionContributions() return result; } + + /// + protected override IList GetSectionMaps() + { + var result = new List(); + + var reader = _sectionMapReader.Fork(); + + ushort count = reader.ReadUInt16(); + ushort logCount = reader.ReadUInt16(); + + for (int i = 0; i < count; i++) + result.Add(SectionMap.FromReader(ref reader)); + + return result; + } } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs index 280266228..45804e09e 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -93,4 +93,24 @@ public void ReadSectionContributions() (24, 0), (10, 0), (14, 0), (2, 0), (31, 0), (3, 0), (3, 0) }, dbiStream.SectionContributions.Select(x => (x.ModuleIndex, x.DataCrc))); } + + [Fact] + public void ReadSectionMap() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); + + Assert.Equal(new (ushort, ushort, ushort, ushort, ushort, ushort, uint, uint)[] + { + (0x010d, 0x0000, 0x0000, 0x0001, 0xffff, 0xffff, 0x00000000, 0x00000ce8), + (0x0109, 0x0000, 0x0000, 0x0002, 0xffff, 0xffff, 0x00000000, 0x00000834), + (0x010b, 0x0000, 0x0000, 0x0003, 0xffff, 0xffff, 0x00000000, 0x00000394), + (0x0109, 0x0000, 0x0000, 0x0004, 0xffff, 0xffff, 0x00000000, 0x000000f8), + (0x0109, 0x0000, 0x0000, 0x0005, 0xffff, 0xffff, 0x00000000, 0x0000013c), + (0x0208, 0x0000, 0x0000, 0x0000, 0xffff, 0xffff, 0x00000000, 0xffffffff), + }, + dbiStream.SectionMaps.Select(m => ((ushort) + m.Attributes, m.LogicalOverlayNumber, m.Group, m.Frame, + m.SectionName, m.ClassName, m.Offset, m.SectionLength))); + } } From 3573e6360c524dc423aa2dd9720423da7105887e Mon Sep 17 00:00:00 2001 From: Washi Date: Tue, 21 Jun 2022 11:43:30 +0200 Subject: [PATCH 08/16] Add basic read support for typeservermap substream. --- .../Metadata/Dbi/DbiStream.cs | 477 +++++++++--------- .../Metadata/Dbi/SerializedDbiStream.cs | 233 ++++----- 2 files changed, 372 insertions(+), 338 deletions(-) diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index 4a0e490fe..926f1d91c 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -1,225 +1,252 @@ -using System.Collections.Generic; -using System.Threading; -using AsmResolver.IO; -using AsmResolver.PE.File.Headers; - -namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; - -/// -/// Represents the DBI Stream (also known as the Debug Information stream). -/// -public class DbiStream : SegmentBase -{ - /// - /// Gets the default fixed MSF stream index for the DBI stream. - /// - public const int StreamIndex = 3; - - private IList? _modules; - private IList? _sectionContributions; - private IList? _sectionMaps; - - /// - /// Gets or sets the version signature assigned to the DBI stream. - /// - /// - /// This value should always be -1 for valid PDB files. - /// - public int VersionSignature - { - get; - set; - } = -1; - - /// - /// Gets or sets the version number of the DBI header. - /// - /// - /// Modern tooling only recognize the VC7.0 file format. - /// - public DbiStreamVersion VersionHeader - { - get; - set; - } = DbiStreamVersion.V70; - - /// - /// Gets or sets the number of times the DBI stream has been written. - /// - public uint Age - { - get; - set; - } = 1; - - /// - /// Gets or sets the MSF stream index of the Global Symbol Stream. - /// - public ushort GlobalStreamIndex - { - get; - set; - } - - /// - /// Gets or sets a bitfield containing the major and minor version of the toolchain that was used to build the program. - /// - public ushort BuildNumber - { - get; - set; - } - - /// - /// Gets or sets the MSF stream index of the Public Symbol Stream. - /// - public ushort PublicStreamIndex - { - get; - set; - } - - /// - /// Gets or sets the version number of mspdbXXXX.dll that was used to produce this PDB file. - /// - public ushort PdbDllVersion - { - get; - set; - } - - /// - /// Gets or sets the MSF stream index of the Symbol Record Stream. - /// - public ushort SymbolRecordStreamIndex - { - get; - set; - } - - /// - /// Unknown. - /// - public ushort PdbDllRbld - { - get; - set; - } - - /// - /// Gets or sets the MSF stream index of the MFC type server. - /// - public uint MfcTypeServerIndex - { - get; - set; - } - - /// - /// Gets or sets attributes associated to the DBI stream. - /// - public DbiAttributes Attributes - { - get; - set; - } - - /// - /// Gets or sets the machine type the program was compiled for. - /// - public MachineType Machine - { - get; - set; - } - - /// - /// Gets a collection of modules (object files) that were linked together into the program. - /// - public IList Modules - { - get - { - if (_modules is null) - Interlocked.CompareExchange(ref _modules, GetModules(), null); - return _modules; - } - } - - /// - /// Gets a collection of section contributions describing the layout of the sections of the final executable file. - /// - public IList SectionContributions - { - get - { - if (_sectionContributions is null) - Interlocked.CompareExchange(ref _sectionContributions, GetSectionContributions(), null); - return _sectionContributions; - } - } - - /// - /// Gets a collection of section mappings stored in the section mapping sub stream. - /// - /// - /// The exact purpose of this is unknown, but it seems to be always containing a copy of the sections in the final - /// executable file. - /// - public IList SectionMaps - { - get - { - if (_sectionMaps is null) - Interlocked.CompareExchange(ref _sectionMaps, GetSectionMaps(), null); - return _sectionMaps; - - } - } - - /// - /// Obtains the list of module descriptors. - /// - /// The module descriptors - /// - /// This method is called upon initialization of the property. - /// - protected virtual IList GetModules() => new List(); - - /// - /// Obtains the list of section contributions. - /// - /// The section contributions. - /// - /// This method is called upon initialization of the property. - /// - protected virtual IList GetSectionContributions() => new List(); - - /// - /// Obtains the list of section maps. - /// - /// The section maps. - /// - /// This method is called upon initialization of the property. - /// - protected virtual IList GetSectionMaps() => new List(); - - /// - /// Reads a single DBI stream from the provided input stream. - /// - /// The input stream. - /// The parsed DBI stream. - public static DbiStream FromReader(BinaryStreamReader reader) => new SerializedDbiStream(reader); - - /// - public override uint GetPhysicalSize() - { - throw new System.NotImplementedException(); - } - - /// - public override void Write(IBinaryStreamWriter writer) - { - throw new System.NotImplementedException(); - } -} +using System.Collections.Generic; +using System.Threading; +using AsmResolver.IO; +using AsmResolver.PE.File.Headers; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Represents the DBI Stream (also known as the Debug Information stream). +/// +public class DbiStream : SegmentBase +{ + /// + /// Gets the default fixed MSF stream index for the DBI stream. + /// + public const int StreamIndex = 3; + + private IList? _modules; + private IList? _sectionContributions; + private IList? _sectionMaps; + private readonly LazyVariable _typeServerMap; + + public DbiStream() + { + _typeServerMap = new LazyVariable(GetTypeServerMap); + } + + /// + /// Gets or sets the version signature assigned to the DBI stream. + /// + /// + /// This value should always be -1 for valid PDB files. + /// + public int VersionSignature + { + get; + set; + } = -1; + + /// + /// Gets or sets the version number of the DBI header. + /// + /// + /// Modern tooling only recognize the VC7.0 file format. + /// + public DbiStreamVersion VersionHeader + { + get; + set; + } = DbiStreamVersion.V70; + + /// + /// Gets or sets the number of times the DBI stream has been written. + /// + public uint Age + { + get; + set; + } = 1; + + /// + /// Gets or sets the MSF stream index of the Global Symbol Stream. + /// + public ushort GlobalStreamIndex + { + get; + set; + } + + /// + /// Gets or sets a bitfield containing the major and minor version of the toolchain that was used to build the program. + /// + public ushort BuildNumber + { + get; + set; + } + + /// + /// Gets or sets the MSF stream index of the Public Symbol Stream. + /// + public ushort PublicStreamIndex + { + get; + set; + } + + /// + /// Gets or sets the version number of mspdbXXXX.dll that was used to produce this PDB file. + /// + public ushort PdbDllVersion + { + get; + set; + } + + /// + /// Gets or sets the MSF stream index of the Symbol Record Stream. + /// + public ushort SymbolRecordStreamIndex + { + get; + set; + } + + /// + /// Unknown. + /// + public ushort PdbDllRbld + { + get; + set; + } + + /// + /// Gets or sets the MSF stream index of the MFC type server. + /// + public uint MfcTypeServerIndex + { + get; + set; + } + + /// + /// Gets or sets attributes associated to the DBI stream. + /// + public DbiAttributes Attributes + { + get; + set; + } + + /// + /// Gets or sets the machine type the program was compiled for. + /// + public MachineType Machine + { + get; + set; + } + + /// + /// Gets a collection of modules (object files) that were linked together into the program. + /// + public IList Modules + { + get + { + if (_modules is null) + Interlocked.CompareExchange(ref _modules, GetModules(), null); + return _modules; + } + } + + /// + /// Gets a collection of section contributions describing the layout of the sections of the final executable file. + /// + public IList SectionContributions + { + get + { + if (_sectionContributions is null) + Interlocked.CompareExchange(ref _sectionContributions, GetSectionContributions(), null); + return _sectionContributions; + } + } + + /// + /// Gets a collection of section mappings stored in the section mapping sub stream. + /// + /// + /// The exact purpose of this is unknown, but it seems to be always containing a copy of the sections in the final + /// executable file. + /// + public IList SectionMaps + { + get + { + if (_sectionMaps is null) + Interlocked.CompareExchange(ref _sectionMaps, GetSectionMaps(), null); + return _sectionMaps; + + } + } + + /// + /// Gets or sets the contents of the type server map sub stream. + /// + /// + /// The exact purpose of this sub stream is unknown. + /// + public ISegment TypeServerMap + { + get => _typeServerMap.Value; + set => _typeServerMap.Value = value; + } + + /// + /// Reads a single DBI stream from the provided input stream. + /// + /// The input stream. + /// The parsed DBI stream. + public static DbiStream FromReader(BinaryStreamReader reader) => new SerializedDbiStream(reader); + + /// + /// Obtains the list of module descriptors. + /// + /// The module descriptors + /// + /// This method is called upon initialization of the property. + /// + protected virtual IList GetModules() => new List(); + + /// + /// Obtains the list of section contributions. + /// + /// The section contributions. + /// + /// This method is called upon initialization of the property. + /// + protected virtual IList GetSectionContributions() => new List(); + + /// + /// Obtains the list of section maps. + /// + /// The section maps. + /// + /// This method is called upon initialization of the property. + /// + protected virtual IList GetSectionMaps() => new List(); + + /// + /// Obtains the contents of the type server map sub stream. + /// + /// The contents of the sub stream. + /// + /// This method is called upon initialization of the property. + /// + protected virtual ISegment? GetTypeServerMap() => null; + + /// + public override uint GetPhysicalSize() + { + throw new System.NotImplementedException(); + } + + /// + public override void Write(IBinaryStreamWriter writer) + { + throw new System.NotImplementedException(); + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index 6e0219b57..6fcc68cc4 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -1,113 +1,120 @@ -using System.Collections.Generic; -using AsmResolver.IO; -using AsmResolver.PE.File.Headers; - -namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; - -/// -/// Implements a DBI stream that pulls its data from an input stream. -/// -public class SerializedDbiStream : DbiStream -{ - private readonly BinaryStreamReader _moduleInfoReader; - private readonly BinaryStreamReader _sectionContributionReader; - private readonly BinaryStreamReader _sectionMapReader; - private readonly BinaryStreamReader _sourceInfoReader; - private readonly BinaryStreamReader _typeServerMapReader; - private readonly BinaryStreamReader _optionalDebugHeaderReader; - private readonly BinaryStreamReader _ecReader; - - /// - /// Parses a DBI stream from an input stream reader. - /// - /// The input stream. - public SerializedDbiStream(BinaryStreamReader reader) - { - VersionSignature = reader.ReadInt32(); - VersionHeader = (DbiStreamVersion) reader.ReadUInt32(); - Age = reader.ReadUInt32(); - GlobalStreamIndex = reader.ReadUInt16(); - BuildNumber = reader.ReadUInt16(); - PublicStreamIndex = reader.ReadUInt16(); - PdbDllVersion = reader.ReadUInt16(); - SymbolRecordStreamIndex = reader.ReadUInt16(); - PdbDllRbld = reader.ReadUInt16(); - - uint moduleInfoSize = reader.ReadUInt32(); - uint sectionContributionSize = reader.ReadUInt32(); - uint sectionMapSize = reader.ReadUInt32(); - uint sourceInfoSize = reader.ReadUInt32(); - uint typeServerMapSize = reader.ReadUInt32(); - - MfcTypeServerIndex = reader.ReadUInt32(); - - uint optionalDebugHeaderSize = reader.ReadUInt32(); - uint ecSize = reader.ReadUInt32(); - - Attributes = (DbiAttributes) reader.ReadUInt16(); - Machine = (MachineType) reader.ReadUInt16(); - - _ = reader.ReadUInt32(); - - _moduleInfoReader = reader.ForkRelative(reader.RelativeOffset, moduleInfoSize); - reader.Offset += moduleInfoSize; - _sectionContributionReader = reader.ForkRelative(reader.RelativeOffset, sectionContributionSize); - reader.Offset += sectionContributionSize; - _sectionMapReader = reader.ForkRelative(reader.RelativeOffset, sectionMapSize); - reader.Offset += sectionMapSize; - _sourceInfoReader = reader.ForkRelative(reader.RelativeOffset, sourceInfoSize); - reader.Offset += sourceInfoSize; - _typeServerMapReader = reader.ForkRelative(reader.RelativeOffset, typeServerMapSize); - reader.Offset += typeServerMapSize; - _ecReader = reader.ForkRelative(reader.RelativeOffset, ecSize); - reader.Offset += ecSize; - _optionalDebugHeaderReader = reader.ForkRelative(reader.RelativeOffset, optionalDebugHeaderSize); - reader.Offset += optionalDebugHeaderSize; - } - - /// - protected override IList GetModules() - { - var result = new List(); - - var reader = _moduleInfoReader.Fork(); - while (reader.CanRead(1)) - result.Add(ModuleDescriptor.FromReader(ref reader)); - - return result; - } - - /// - protected override IList GetSectionContributions() - { - var result = new List(); - - var reader = _sectionContributionReader.Fork(); - var version = (SectionContributionStreamVersion) reader.ReadUInt32(); - - while (reader.CanRead(1)) - { - result.Add(SectionContribution.FromReader(ref reader)); - if (version == SectionContributionStreamVersion.V2) - reader.ReadUInt32(); - } - - return result; - } - - /// - protected override IList GetSectionMaps() - { - var result = new List(); - - var reader = _sectionMapReader.Fork(); - - ushort count = reader.ReadUInt16(); - ushort logCount = reader.ReadUInt16(); - - for (int i = 0; i < count; i++) - result.Add(SectionMap.FromReader(ref reader)); - - return result; - } -} +using System.Collections.Generic; +using AsmResolver.IO; +using AsmResolver.PE.File.Headers; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Implements a DBI stream that pulls its data from an input stream. +/// +public class SerializedDbiStream : DbiStream +{ + private readonly BinaryStreamReader _moduleInfoReader; + private readonly BinaryStreamReader _sectionContributionReader; + private readonly BinaryStreamReader _sectionMapReader; + private readonly BinaryStreamReader _sourceInfoReader; + private readonly BinaryStreamReader _typeServerMapReader; + private readonly BinaryStreamReader _optionalDebugHeaderReader; + private readonly BinaryStreamReader _ecReader; + + /// + /// Parses a DBI stream from an input stream reader. + /// + /// The input stream. + public SerializedDbiStream(BinaryStreamReader reader) + { + VersionSignature = reader.ReadInt32(); + VersionHeader = (DbiStreamVersion) reader.ReadUInt32(); + Age = reader.ReadUInt32(); + GlobalStreamIndex = reader.ReadUInt16(); + BuildNumber = reader.ReadUInt16(); + PublicStreamIndex = reader.ReadUInt16(); + PdbDllVersion = reader.ReadUInt16(); + SymbolRecordStreamIndex = reader.ReadUInt16(); + PdbDllRbld = reader.ReadUInt16(); + + uint moduleInfoSize = reader.ReadUInt32(); + uint sectionContributionSize = reader.ReadUInt32(); + uint sectionMapSize = reader.ReadUInt32(); + uint sourceInfoSize = reader.ReadUInt32(); + uint typeServerMapSize = reader.ReadUInt32(); + + MfcTypeServerIndex = reader.ReadUInt32(); + + uint optionalDebugHeaderSize = reader.ReadUInt32(); + uint ecSize = reader.ReadUInt32(); + + Attributes = (DbiAttributes) reader.ReadUInt16(); + Machine = (MachineType) reader.ReadUInt16(); + + _ = reader.ReadUInt32(); + + _moduleInfoReader = reader.ForkRelative(reader.RelativeOffset, moduleInfoSize); + reader.Offset += moduleInfoSize; + _sectionContributionReader = reader.ForkRelative(reader.RelativeOffset, sectionContributionSize); + reader.Offset += sectionContributionSize; + _sectionMapReader = reader.ForkRelative(reader.RelativeOffset, sectionMapSize); + reader.Offset += sectionMapSize; + _sourceInfoReader = reader.ForkRelative(reader.RelativeOffset, sourceInfoSize); + reader.Offset += sourceInfoSize; + _typeServerMapReader = reader.ForkRelative(reader.RelativeOffset, typeServerMapSize); + reader.Offset += typeServerMapSize; + _ecReader = reader.ForkRelative(reader.RelativeOffset, ecSize); + reader.Offset += ecSize; + _optionalDebugHeaderReader = reader.ForkRelative(reader.RelativeOffset, optionalDebugHeaderSize); + reader.Offset += optionalDebugHeaderSize; + } + + /// + protected override IList GetModules() + { + var result = new List(); + + var reader = _moduleInfoReader.Fork(); + while (reader.CanRead(1)) + result.Add(ModuleDescriptor.FromReader(ref reader)); + + return result; + } + + /// + protected override IList GetSectionContributions() + { + var result = new List(); + + var reader = _sectionContributionReader.Fork(); + var version = (SectionContributionStreamVersion) reader.ReadUInt32(); + + while (reader.CanRead(1)) + { + result.Add(SectionContribution.FromReader(ref reader)); + if (version == SectionContributionStreamVersion.V2) + reader.ReadUInt32(); + } + + return result; + } + + /// + protected override IList GetSectionMaps() + { + var result = new List(); + + var reader = _sectionMapReader.Fork(); + + ushort count = reader.ReadUInt16(); + ushort logCount = reader.ReadUInt16(); + + for (int i = 0; i < count; i++) + result.Add(SectionMap.FromReader(ref reader)); + + return result; + } + + /// + protected override ISegment? GetTypeServerMap() + { + var reader = _typeServerMapReader; + return reader.ReadSegment(reader.Length); + } +} From 0f6032c71f67044877004288d70c9d802249a7bd Mon Sep 17 00:00:00 2001 From: Washi Date: Tue, 21 Jun 2022 11:46:47 +0200 Subject: [PATCH 09/16] Add basic read support for ec substream. --- .../Metadata/Dbi/DbiStream.cs | 41 +++++++++++++++---- .../Metadata/Dbi/SerializedDbiStream.cs | 17 ++++++-- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index 926f1d91c..6ebd15df3 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -18,11 +18,13 @@ public class DbiStream : SegmentBase private IList? _modules; private IList? _sectionContributions; private IList? _sectionMaps; - private readonly LazyVariable _typeServerMap; + private readonly LazyVariable _typeServerMapStream; + private readonly LazyVariable _ecStream; public DbiStream() { - _typeServerMap = new LazyVariable(GetTypeServerMap); + _typeServerMapStream = new LazyVariable(GetTypeServerMapStream); + _ecStream = new LazyVariable(GetECStream); } /// @@ -187,12 +189,26 @@ public IList SectionMaps /// Gets or sets the contents of the type server map sub stream. /// /// - /// The exact purpose of this sub stream is unknown. + /// The exact purpose and layout of this sub stream is unknown, hence this property exposes the stream as + /// a raw segment. /// - public ISegment TypeServerMap + public ISegment TypeServerMapStream { - get => _typeServerMap.Value; - set => _typeServerMap.Value = value; + get => _typeServerMapStream.Value; + set => _typeServerMapStream.Value = value; + } + + /// + /// Gets or sets the contents of the Edit-and-Continue sub stream. + /// + /// + /// The exact purpose and layout of this sub stream is unknown, hence this property exposes the stream as + /// a raw segment. + /// + public ISegment ECStream + { + get => _ecStream.Value; + set => _ecStream.Value = value; } /// @@ -234,9 +250,18 @@ public ISegment TypeServerMap /// /// The contents of the sub stream. /// - /// This method is called upon initialization of the property. + /// This method is called upon initialization of the property. + /// + protected virtual ISegment? GetTypeServerMapStream() => null; + + /// + /// Obtains the contents of the EC sub stream. + /// + /// The contents of the sub stream. + /// + /// This method is called upon initialization of the property. /// - protected virtual ISegment? GetTypeServerMap() => null; + protected virtual ISegment? GetECStream() => null; /// public override uint GetPhysicalSize() diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index 6fcc68cc4..3d6bad38c 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -112,9 +112,20 @@ protected override IList GetSectionMaps() } /// - protected override ISegment? GetTypeServerMap() + protected override ISegment? GetTypeServerMapStream() { - var reader = _typeServerMapReader; - return reader.ReadSegment(reader.Length); + var reader = _typeServerMapReader.Fork(); + return reader.Length != 0 + ? reader.ReadSegment(reader.Length) + : null; + } + + /// + protected override ISegment? GetECStream() + { + var reader = _ecReader.Fork(); + return reader.Length != 0 + ? reader.ReadSegment(reader.Length) + : null; } } From fea51df9f864755bf3af353bc44444a7cf94088d Mon Sep 17 00:00:00 2001 From: Washi Date: Tue, 21 Jun 2022 19:58:15 +0200 Subject: [PATCH 10/16] Add read support for source file collections. --- .../Metadata/Dbi/DbiStream.cs | 32 ++++++++++++- .../Metadata/Dbi/SerializedDbiStream.cs | 48 +++++++++++++++++++ .../Metadata/Dbi/SourceFileCollection.cs | 37 ++++++++++++++ .../Metadata/Dbi/DbiStreamTest.cs | 45 +++++++++++++++++ 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SourceFileCollection.cs diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index 6ebd15df3..05495f5b4 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -20,7 +20,11 @@ public class DbiStream : SegmentBase private IList? _sectionMaps; private readonly LazyVariable _typeServerMapStream; private readonly LazyVariable _ecStream; + private IList? _sourceFiles; + /// + /// Creates a new empty DBI stream. + /// public DbiStream() { _typeServerMapStream = new LazyVariable(GetTypeServerMapStream); @@ -181,7 +185,6 @@ public IList SectionMaps if (_sectionMaps is null) Interlocked.CompareExchange(ref _sectionMaps, GetSectionMaps(), null); return _sectionMaps; - } } @@ -211,6 +214,24 @@ public ISegment ECStream set => _ecStream.Value = value; } + /// + /// Gets a collection of source files assigned to each module in . + /// + /// + /// Every collection of source files within this list corresponds to exactly the module within + /// at the same index. For example, the first source file list in this collection is the source file list of the + /// first module. + /// + public IList SourceFiles + { + get + { + if (_sourceFiles is null) + Interlocked.CompareExchange(ref _sourceFiles, GetSourceFiles(), null); + return _sourceFiles; + } + } + /// /// Reads a single DBI stream from the provided input stream. /// @@ -263,6 +284,15 @@ public ISegment ECStream /// protected virtual ISegment? GetECStream() => null; + /// + /// Obtains a table that assigns a list of source files to every module referenced in the PDB file. + /// + /// The table. + /// + /// This method is called upon initialization of the property. + /// + protected virtual IList GetSourceFiles() => new List(); + /// public override uint GetPhysicalSize() { diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index 3d6bad38c..8e1650274 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -128,4 +128,52 @@ protected override IList GetSectionMaps() ? reader.ReadSegment(reader.Length) : null; } + + /// + protected override IList GetSourceFiles() + { + var result = new List(); + + var reader = _sourceInfoReader.Fork(); + + ushort moduleCount = reader.ReadUInt16(); + ushort sourceFileCount = reader.ReadUInt16(); + + // Read module indices. + ushort[] moduleIndices = new ushort[moduleCount]; + for (int i = 0; i < moduleCount; i++) + moduleIndices[i] = reader.ReadUInt16(); + + // Read module source file counts. + int actualFileCount = 0; + ushort[] moduleFileCounts = new ushort[moduleCount]; + for (int i = 0; i < moduleCount; i++) + { + ushort count = reader.ReadUInt16(); + moduleFileCounts[i] = count; + actualFileCount += count; + } + + // Scope on the name buffer. + var stringReaderBuffer = reader.ForkRelative((uint) (reader.RelativeOffset + actualFileCount * sizeof(uint))); + + // Construct source file lists. + for (int i = 0; i < moduleCount; i++) + { + var files = new SourceFileCollection(moduleIndices[i]); + ushort fileCount = moduleFileCounts[i]; + + // Read all file paths for this module. + for (int j = 0; j < fileCount; j++) + { + uint nameOffset = reader.ReadUInt32(); + var nameReader = stringReaderBuffer.ForkRelative(nameOffset); + files.Add(new Utf8String(nameReader.ReadBytesUntil(0, false))); + } + + result.Add(files); + } + + return result; + } } diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SourceFileCollection.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SourceFileCollection.cs new file mode 100644 index 000000000..b7ef673e6 --- /dev/null +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SourceFileCollection.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Represents a collection of file paths used as input source code for a single module. +/// +public class SourceFileCollection : List +{ + /// + /// Creates a new empty source file collection. + /// + public SourceFileCollection() + { + } + + /// + /// Creates a new empty source file collection. + /// + /// The original module index for which this collection was compiled. + public SourceFileCollection(uint originalModuleIndex) + { + OriginalModuleIndex = originalModuleIndex; + } + + /// + /// Gets the original module index for which this collection was compiled (if available). + /// + /// + /// The exact purpose of this number is unclear, as this number cannot be reliably used as an index within the + /// DBI stream's module list. Use the index of this list within instead. + /// + public uint OriginalModuleIndex + { + get; + } +} diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs index 45804e09e..85c4cd836 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using AsmResolver.PE.File.Headers; using AsmResolver.Symbols.Pdb.Metadata.Dbi; @@ -113,4 +114,48 @@ public void ReadSectionMap() m.Attributes, m.LogicalOverlayNumber, m.Group, m.Frame, m.SectionName, m.ClassName, m.Offset, m.SectionLength))); } + + [Fact] + public void ReadSourceFiles() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); + + string[][] firstThreeActualFileLists = dbiStream.SourceFiles + .Take(3) + .Select(x => x + .Select(y => y.ToString()) + .ToArray() + ).ToArray(); + + Assert.Equal(new[] + { + Array.Empty(), + new[] + { + @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\pch.h", + @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\dllmain.cpp", + @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\Release\SimpleDll.pch", + }, + new[] + { + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winuser.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\basetsd.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winbase.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\stralign.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\guiddef.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\winerror.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_wstring.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\processthreadsapi.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winnt.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\ctype.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\string.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_memory.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\memoryapi.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_memcpy_s.h", + @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\Release\SimpleDll.pch", + } + }, + firstThreeActualFileLists); + } } From 5f700acdac3bf8186b5fe5afb1fd4d530878aa49 Mon Sep 17 00:00:00 2001 From: Washi Date: Tue, 21 Jun 2022 20:10:33 +0200 Subject: [PATCH 11/16] Add read support extra dbg stream indices. --- .../Metadata/Dbi/DbiStream.cs | 23 +++++++++++++++++++ .../Metadata/Dbi/SerializedDbiStream.cs | 19 +++++++++++---- .../Metadata/Dbi/DbiStreamTest.cs | 11 +++++++++ 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index 05495f5b4..9744d852e 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -21,6 +21,7 @@ public class DbiStream : SegmentBase private readonly LazyVariable _typeServerMapStream; private readonly LazyVariable _ecStream; private IList? _sourceFiles; + private IList? _extraStreamIndices; /// /// Creates a new empty DBI stream. @@ -232,6 +233,19 @@ public IList SourceFiles } } + /// + /// Gets a collection of indices referring to additional debug streams in the MSF file. + /// + public IList ExtraStreamIndices + { + get + { + if (_extraStreamIndices is null) + Interlocked.CompareExchange(ref _extraStreamIndices, GetExtraStreamIndices(), null); + return _extraStreamIndices; + } + } + /// /// Reads a single DBI stream from the provided input stream. /// @@ -293,6 +307,15 @@ public IList SourceFiles /// protected virtual IList GetSourceFiles() => new List(); + /// + /// Obtains the list of indices referring to additional debug streams in the MSF file. + /// + /// The list of indices. + /// + /// This method is called upon initialization of the property. + /// + protected virtual IList GetExtraStreamIndices() => new List(); + /// public override uint GetPhysicalSize() { diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index 8e1650274..4c6fd2632 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -98,13 +98,12 @@ protected override IList GetSectionContributions() /// protected override IList GetSectionMaps() { - var result = new List(); - var reader = _sectionMapReader.Fork(); ushort count = reader.ReadUInt16(); ushort logCount = reader.ReadUInt16(); + var result = new List(count); for (int i = 0; i < count; i++) result.Add(SectionMap.FromReader(ref reader)); @@ -132,8 +131,6 @@ protected override IList GetSectionMaps() /// protected override IList GetSourceFiles() { - var result = new List(); - var reader = _sourceInfoReader.Fork(); ushort moduleCount = reader.ReadUInt16(); @@ -158,6 +155,7 @@ protected override IList GetSourceFiles() var stringReaderBuffer = reader.ForkRelative((uint) (reader.RelativeOffset + actualFileCount * sizeof(uint))); // Construct source file lists. + var result = new List(moduleCount); for (int i = 0; i < moduleCount; i++) { var files = new SourceFileCollection(moduleIndices[i]); @@ -176,4 +174,17 @@ protected override IList GetSourceFiles() return result; } + + /// + protected override IList GetExtraStreamIndices() + { + var reader = _optionalDebugHeaderReader.Fork(); + + var result = new List((int) (reader.Length / sizeof(ushort))); + + while (reader.CanRead(sizeof(ushort))) + result.Add(reader.ReadUInt16()); + + return result; + } } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs index 85c4cd836..179173850 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -158,4 +158,15 @@ public void ReadSourceFiles() }, firstThreeActualFileLists); } + + [Fact] + public void ReadExtraDebugIndices() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); + Assert.Equal(new ushort[] + { + 0x7, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xB, 0xFFFF, 0xFFFF, 0xFFFF, 0xD, 0xFFFF + }, dbiStream.ExtraStreamIndices); + } } From 6f169b5742a7f57eaa8bd938591a91347de2bb63 Mon Sep 17 00:00:00 2001 From: Washi Date: Fri, 24 Jun 2022 16:01:23 +0200 Subject: [PATCH 12/16] Add write support for DBI stream. --- .../Metadata/Dbi/DbiStream.cs | 253 ++++++++++- .../Metadata/Dbi/ModuleDescriptor.cs | 414 +++++++++--------- .../Metadata/Dbi/SectionContribution.cs | 253 +++++------ .../Metadata/Dbi/SectionMap.cs | 259 +++++------ .../Metadata/Dbi/DbiStreamTest.cs | 368 ++++++++-------- 5 files changed, 907 insertions(+), 640 deletions(-) diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index 9744d852e..e9180fb4d 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -1,4 +1,6 @@ using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Threading; using AsmResolver.IO; using AsmResolver.PE.File.Headers; @@ -18,8 +20,8 @@ public class DbiStream : SegmentBase private IList? _modules; private IList? _sectionContributions; private IList? _sectionMaps; - private readonly LazyVariable _typeServerMapStream; - private readonly LazyVariable _ecStream; + private readonly LazyVariable _typeServerMapStream; + private readonly LazyVariable _ecStream; private IList? _sourceFiles; private IList? _extraStreamIndices; @@ -28,8 +30,8 @@ public class DbiStream : SegmentBase /// public DbiStream() { - _typeServerMapStream = new LazyVariable(GetTypeServerMapStream); - _ecStream = new LazyVariable(GetECStream); + _typeServerMapStream = new LazyVariable(GetTypeServerMapStream); + _ecStream = new LazyVariable(GetECStream); } /// @@ -196,7 +198,7 @@ public IList SectionMaps /// The exact purpose and layout of this sub stream is unknown, hence this property exposes the stream as /// a raw segment. /// - public ISegment TypeServerMapStream + public ISegment? TypeServerMapStream { get => _typeServerMapStream.Value; set => _typeServerMapStream.Value = value; @@ -209,7 +211,7 @@ public ISegment TypeServerMapStream /// The exact purpose and layout of this sub stream is unknown, hence this property exposes the stream as /// a raw segment. /// - public ISegment ECStream + public ISegment? ECStream { get => _ecStream.Value; set => _ecStream.Value = value; @@ -319,12 +321,247 @@ public IList ExtraStreamIndices /// public override uint GetPhysicalSize() { - throw new System.NotImplementedException(); + return GetHeaderSize() + + GetModuleStreamSize() + + GetSectionContributionStreamSize() + + GetSectionMapStreamSize() + + GetSourceInfoStreamSize() + + GetTypeServerMapStreamSize() + + GetECStreamSize() + + GetOptionalDebugStreamSize() + ; + } + + private static uint GetHeaderSize() + { + return sizeof(int) // VersionSignature + + sizeof(DbiStreamVersion) // VersionHeader + + sizeof(uint) // Age + + sizeof(ushort) // GlobalStreamIndex + + sizeof(ushort) // BuildNumber + + sizeof(ushort) // PublicStreamIndex + + sizeof(ushort) // PdbDllVersion + + sizeof(ushort) // SymbolRecordStreamIndex + + sizeof(ushort) // PdbDllRbld + + sizeof(uint) // ModuleInfoSize + + sizeof(uint) // SectionContributionSize + + sizeof(uint) // SectionMapSize + + sizeof(uint) // SourceInfoSize + + sizeof(uint) // TypeServerMapSize + + sizeof(uint) // MfcTypeServerIndex + + sizeof(uint) // OptionalDebugStreamSize + + sizeof(uint) // ECStreamSize + + sizeof(DbiAttributes) // Attributes + + sizeof(MachineType) // MachineType + + sizeof(uint) // Padding + ; + } + + private uint GetModuleStreamSize() + { + return ((uint) Modules.Sum(m => m.GetPhysicalSize())).Align(sizeof(uint)); + } + + private uint GetSectionContributionStreamSize() + { + return sizeof(uint) // version + + SectionContribution.EntrySize * (uint) SectionContributions.Count; + } + + private uint GetSectionMapStreamSize() + { + return sizeof(ushort) // Count + + sizeof(ushort) // LogCount + + SectionMap.EntrySize * (uint) SectionMaps.Count; + } + + private uint GetSourceInfoStreamSize() + { + uint totalFileCount = 0; + uint nameBufferSize = 0; + var stringOffsets = new Dictionary(); + + // Simulate the construction of name buffer + for (int i = 0; i < SourceFiles.Count; i++) + { + var collection = SourceFiles[i]; + totalFileCount += (uint) collection.Count; + + for (int j = 0; j < collection.Count; j++) + { + // If name is not added yet, "append" to the name buffer. + var name = collection[j]; + if (!stringOffsets.ContainsKey(name)) + { + stringOffsets[name] = nameBufferSize; + nameBufferSize += (uint) name.ByteCount + 1u; + } + } + } + + return (sizeof(ushort) // ModuleCount + + sizeof(ushort) // SourceFileCount + + sizeof(ushort) * (uint) SourceFiles.Count // ModuleIndices + + sizeof(ushort) * (uint) SourceFiles.Count // SourceFileCounts + + sizeof(uint) * totalFileCount // NameOffsets + + nameBufferSize // NameBuffer + ).Align(4); + } + + private uint GetTypeServerMapStreamSize() + { + return TypeServerMapStream?.GetPhysicalSize().Align(sizeof(uint)) ?? 0u; + } + + private uint GetOptionalDebugStreamSize() + { + return (uint) (ExtraStreamIndices.Count * sizeof(ushort)); + } + + private uint GetECStreamSize() + { + return ECStream?.GetPhysicalSize() ?? 0u; } /// public override void Write(IBinaryStreamWriter writer) { - throw new System.NotImplementedException(); + WriteHeader(writer); + WriteModuleStream(writer); + WriteSectionContributionStream(writer); + WriteSectionMapStream(writer); + WriteSourceInfoStream(writer); + WriteTypeServerMapStream(writer); + WriteECStream(writer); + WriteOptionalDebugStream(writer); + } + + private void WriteHeader(IBinaryStreamWriter writer) + { + writer.WriteInt32(VersionSignature); + writer.WriteUInt32((uint) VersionHeader); + writer.WriteUInt32(Age); + writer.WriteUInt16(GlobalStreamIndex); + writer.WriteUInt16(BuildNumber); + writer.WriteUInt16(PublicStreamIndex); + writer.WriteUInt16(PdbDllVersion); + writer.WriteUInt16(SymbolRecordStreamIndex); + writer.WriteUInt16(PdbDllRbld); + + writer.WriteUInt32(GetModuleStreamSize()); + writer.WriteUInt32(GetSectionContributionStreamSize()); + writer.WriteUInt32(GetSectionMapStreamSize()); + writer.WriteUInt32(GetSourceInfoStreamSize()); + writer.WriteUInt32(GetTypeServerMapStreamSize()); + + writer.WriteUInt32(MfcTypeServerIndex); + + writer.WriteUInt32(GetOptionalDebugStreamSize()); + writer.WriteUInt32(GetECStreamSize()); + + writer.WriteUInt16((ushort) Attributes); + writer.WriteUInt16((ushort) Machine); + + writer.WriteUInt32(0); + } + + private void WriteModuleStream(IBinaryStreamWriter writer) + { + var modules = Modules; + for (int i = 0; i < modules.Count; i++) + modules[i].Write(writer); + + writer.Align(sizeof(uint)); } + + private void WriteSectionContributionStream(IBinaryStreamWriter writer) + { + // TODO: make customizable + writer.WriteUInt32((uint) SectionContributionStreamVersion.Ver60); + + var contributions = SectionContributions; + for (int i = 0; i < contributions.Count; i++) + contributions[i].Write(writer); + + writer.Align(sizeof(uint)); + } + + private void WriteSectionMapStream(IBinaryStreamWriter writer) + { + var maps = SectionMaps; + + // Count and LogCount. + writer.WriteUInt16((ushort) maps.Count); + writer.WriteUInt16((ushort) maps.Count); + + // Entries. + for (int i = 0; i < maps.Count; i++) + maps[i].Write(writer); + + writer.Align(sizeof(uint)); + } + + private void WriteSourceInfoStream(IBinaryStreamWriter writer) + { + var sourceFiles = SourceFiles; + int totalFileCount = sourceFiles.Sum(x => x.Count); + + // Module and total file count (truncated to 16 bits) + writer.WriteUInt16((ushort) (sourceFiles.Count & 0xFFFF)); + writer.WriteUInt16((ushort) (totalFileCount & 0xFFFF)); + + // Module indices. Unsure if this is correct, but this array does not seem to be really used by the ref impl. + for (ushort i = 0; i < sourceFiles.Count; i++) + writer.WriteUInt16(i); + + // Source file counts. + for (int i = 0; i < sourceFiles.Count; i++) + writer.WriteUInt16((ushort) sourceFiles[i].Count); + + // Build up string buffer and name offset table. + using var stringBuffer = new MemoryStream(); + var stringWriter = new BinaryStreamWriter(stringBuffer); + var stringOffsets = new Dictionary(); + + for (int i = 0; i < sourceFiles.Count; i++) + { + var collection = sourceFiles[i]; + for (int j = 0; j < collection.Count; j++) + { + // If not present already, append to string buffer. + var name = collection[j]; + if (!stringOffsets.TryGetValue(name, out uint offset)) + { + offset = (uint) stringWriter.Offset; + stringOffsets[name] = offset; + stringWriter.WriteBytes(name.GetBytesUnsafe()); + stringWriter.WriteByte(0); + } + + // Write name offset + writer.WriteUInt32(offset); + } + } + + // Write string buffer. + writer.WriteBytes(stringBuffer.ToArray()); + + writer.Align(sizeof(uint)); + } + + private void WriteTypeServerMapStream(IBinaryStreamWriter writer) + { + TypeServerMapStream?.Write(writer); + writer.Align(sizeof(uint)); + } + + private void WriteOptionalDebugStream(IBinaryStreamWriter writer) + { + var extraIndices = ExtraStreamIndices; + + for (int i = 0; i < extraIndices.Count; i++) + writer.WriteUInt16(extraIndices[i]); + } + + private void WriteECStream(IBinaryStreamWriter writer) => ECStream?.Write(writer); } diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs index 968777a1d..5fee61c3e 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs @@ -1,207 +1,207 @@ -using AsmResolver.IO; - -namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; - -/// -/// Represents a reference to a single module (object file) that was linked into a program. -/// -public class ModuleDescriptor : IWritable -{ - /// - /// Gets or sets a description of the section within the final binary which contains code - /// and/or data from this module. - /// - public SectionContribution SectionContribution - { - get; - set; - } = new(); - - /// - /// Gets or sets the attributes assigned to this module descriptor. - /// - public ModuleDescriptorAttributes Attributes - { - get; - set; - } - - /// - /// Gets or sets the index of the type server for this module. - /// - public ushort TypeServerIndex - { - get => (ushort) ((ushort) Attributes >> 8); - set => Attributes = (Attributes & ~ModuleDescriptorAttributes.TsmMask) | (ModuleDescriptorAttributes) (value << 8); - } - - /// - /// Gets or sets the MSF stream index of the stream that the symbols of this module. - /// - public ushort SymbolStreamIndex - { - get; - set; - } - - /// - /// Gets or sets the size of the CodeView data within the module's symbol stream. - /// - public uint SymbolDataSize - { - get; - set; - } - - /// - /// Gets or sets the size of the C11-style CodeView data within the module's symbol stream. - /// - public uint SymbolC11DataSize - { - get; - set; - } - - /// - /// Gets or sets the size of the C13-style CodeView data within the module's symbol stream. - /// - public uint SymbolC13DataSize - { - get; - set; - } - - /// - /// Gets or sets the number of source files that contributed to this module during the compilation. - /// - public ushort SourceFileCount - { - get; - set; - } - - /// - /// Gets or sets the offset in the names buffer of the primary translation unit. - /// - /// - /// For most compilers this value is set to zero. - /// - public uint SourceFileNameIndex - { - get; - set; - } - - /// - /// Gets or sets the offset in the names buffer of the PDB file. - /// - /// - /// For most modules (except the special * LINKER * module) this value is set to zero. - /// - public uint PdbFilePathNameIndex - { - get; - set; - } - - /// - /// Gets or sets the name of the module. - /// - /// - /// This is often a full path to the object file that was passed into link.exe directly, or a string in the - /// form of Import:dll_name - /// - public Utf8String? ModuleName - { - get; - set; - } - - /// - /// Gets or sets the name of the object file name. - /// - /// - /// In the case this module is linked directly passed to link.exe, this is the same as . - /// If the module comes from an archive, this is the full path to that archive. - /// - public Utf8String? ObjectFileName - { - get; - set; - } - - /// - /// Parses a single module descriptor from the provided input stream. - /// - /// The input stream. - /// THe parsed module descriptor. - public static ModuleDescriptor FromReader(ref BinaryStreamReader reader) - { - var result = new ModuleDescriptor(); - - reader.ReadUInt32(); - result.SectionContribution = SectionContribution.FromReader(ref reader); - result.Attributes = (ModuleDescriptorAttributes) reader.ReadUInt16(); - result.SymbolStreamIndex = reader.ReadUInt16(); - result.SymbolDataSize = reader.ReadUInt32(); - result.SymbolC11DataSize = reader.ReadUInt32(); - result.SymbolC13DataSize = reader.ReadUInt32(); - result.SourceFileCount = reader.ReadUInt16(); - reader.ReadUInt16(); - reader.ReadUInt32(); - result.SourceFileNameIndex = reader.ReadUInt32(); - result.PdbFilePathNameIndex = reader.ReadUInt32(); - result.ModuleName = new Utf8String(reader.ReadBytesUntil(0, false)); - result.ObjectFileName = new Utf8String(reader.ReadBytesUntil(0, false)); - reader.Align(4); - - return result; - } - - /// - public uint GetPhysicalSize() - { - return sizeof(uint) // Unused1 - + SectionContribution.GetPhysicalSize() // SectionContribution - + sizeof(ModuleDescriptorAttributes) // Attributes - + sizeof(ushort) // SymbolStreamIndex - + sizeof(uint) // SymbolDataSize - + sizeof(uint) // SymbolC11DataSize - + sizeof(uint) // SymbolC13DataSize - + sizeof(ushort) // SourceFileCount - + sizeof(char) * 2 // Padding - + sizeof(uint) // Unused2 - + sizeof(uint) // SourceFileNameIndex - + sizeof(uint) // PdbFilePathNameIndex - + (uint) (ModuleName?.ByteCount ?? 0) + 1 // ModuleName - + (uint) (ObjectFileName?.ByteCount ?? 0) + 1 // ObjectFileName - ; - } - - /// - public void Write(IBinaryStreamWriter writer) - { - writer.WriteUInt32(0); - SectionContribution.Write(writer); - writer.WriteUInt16((ushort) Attributes); - writer.WriteUInt16(SymbolStreamIndex); - writer.WriteUInt32(SymbolDataSize); - writer.WriteUInt32(SymbolC11DataSize); - writer.WriteUInt32(SymbolC13DataSize); - writer.WriteUInt16(SourceFileCount); - writer.WriteUInt16(0); - writer.WriteUInt32(0); - writer.WriteUInt32(SourceFileNameIndex); - writer.WriteUInt32(PdbFilePathNameIndex); - if (ModuleName is not null) - writer.WriteBytes(ModuleName.GetBytesUnsafe()); - writer.WriteByte(0); - if (ObjectFileName is not null) - writer.WriteBytes(ObjectFileName.GetBytesUnsafe()); - writer.WriteByte(0); - writer.Align(4); - } - - /// - public override string ToString() => ModuleName ?? ObjectFileName ?? "?"; -} +using AsmResolver.IO; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Represents a reference to a single module (object file) that was linked into a program. +/// +public class ModuleDescriptor : IWritable +{ + /// + /// Gets or sets a description of the section within the final binary which contains code + /// and/or data from this module. + /// + public SectionContribution SectionContribution + { + get; + set; + } = new(); + + /// + /// Gets or sets the attributes assigned to this module descriptor. + /// + public ModuleDescriptorAttributes Attributes + { + get; + set; + } + + /// + /// Gets or sets the index of the type server for this module. + /// + public ushort TypeServerIndex + { + get => (ushort) ((ushort) Attributes >> 8); + set => Attributes = (Attributes & ~ModuleDescriptorAttributes.TsmMask) | (ModuleDescriptorAttributes) (value << 8); + } + + /// + /// Gets or sets the MSF stream index of the stream that the symbols of this module. + /// + public ushort SymbolStreamIndex + { + get; + set; + } + + /// + /// Gets or sets the size of the CodeView data within the module's symbol stream. + /// + public uint SymbolDataSize + { + get; + set; + } + + /// + /// Gets or sets the size of the C11-style CodeView data within the module's symbol stream. + /// + public uint SymbolC11DataSize + { + get; + set; + } + + /// + /// Gets or sets the size of the C13-style CodeView data within the module's symbol stream. + /// + public uint SymbolC13DataSize + { + get; + set; + } + + /// + /// Gets or sets the number of source files that contributed to this module during the compilation. + /// + public ushort SourceFileCount + { + get; + set; + } + + /// + /// Gets or sets the offset in the names buffer of the primary translation unit. + /// + /// + /// For most compilers this value is set to zero. + /// + public uint SourceFileNameIndex + { + get; + set; + } + + /// + /// Gets or sets the offset in the names buffer of the PDB file. + /// + /// + /// For most modules (except the special * LINKER * module) this value is set to zero. + /// + public uint PdbFilePathNameIndex + { + get; + set; + } + + /// + /// Gets or sets the name of the module. + /// + /// + /// This is often a full path to the object file that was passed into link.exe directly, or a string in the + /// form of Import:dll_name + /// + public Utf8String? ModuleName + { + get; + set; + } + + /// + /// Gets or sets the name of the object file name. + /// + /// + /// In the case this module is linked directly passed to link.exe, this is the same as . + /// If the module comes from an archive, this is the full path to that archive. + /// + public Utf8String? ObjectFileName + { + get; + set; + } + + /// + /// Parses a single module descriptor from the provided input stream. + /// + /// The input stream. + /// THe parsed module descriptor. + public static ModuleDescriptor FromReader(ref BinaryStreamReader reader) + { + var result = new ModuleDescriptor(); + + reader.ReadUInt32(); + result.SectionContribution = SectionContribution.FromReader(ref reader); + result.Attributes = (ModuleDescriptorAttributes) reader.ReadUInt16(); + result.SymbolStreamIndex = reader.ReadUInt16(); + result.SymbolDataSize = reader.ReadUInt32(); + result.SymbolC11DataSize = reader.ReadUInt32(); + result.SymbolC13DataSize = reader.ReadUInt32(); + result.SourceFileCount = reader.ReadUInt16(); + reader.ReadUInt16(); + reader.ReadUInt32(); + result.SourceFileNameIndex = reader.ReadUInt32(); + result.PdbFilePathNameIndex = reader.ReadUInt32(); + result.ModuleName = new Utf8String(reader.ReadBytesUntil(0, false)); + result.ObjectFileName = new Utf8String(reader.ReadBytesUntil(0, false)); + reader.Align(4); + + return result; + } + + /// + public uint GetPhysicalSize() + { + return (sizeof(uint) // Unused1 + + SectionContribution.GetPhysicalSize() // SectionContribution + + sizeof(ModuleDescriptorAttributes) // Attributes + + sizeof(ushort) // SymbolStreamIndex + + sizeof(uint) // SymbolDataSize + + sizeof(uint) // SymbolC11DataSize + + sizeof(uint) // SymbolC13DataSize + + sizeof(ushort) // SourceFileCount + + sizeof(ushort) // Padding + + sizeof(uint) // Unused2 + + sizeof(uint) // SourceFileNameIndex + + sizeof(uint) // PdbFilePathNameIndex + + (uint) (ModuleName?.ByteCount ?? 0) + 1 // ModuleName + + (uint) (ObjectFileName?.ByteCount ?? 0) + 1 // ObjectFileName + ).Align(4); + } + + /// + public void Write(IBinaryStreamWriter writer) + { + writer.WriteUInt32(0); + SectionContribution.Write(writer); + writer.WriteUInt16((ushort) Attributes); + writer.WriteUInt16(SymbolStreamIndex); + writer.WriteUInt32(SymbolDataSize); + writer.WriteUInt32(SymbolC11DataSize); + writer.WriteUInt32(SymbolC13DataSize); + writer.WriteUInt16(SourceFileCount); + writer.WriteUInt16(0); + writer.WriteUInt32(0); + writer.WriteUInt32(SourceFileNameIndex); + writer.WriteUInt32(PdbFilePathNameIndex); + if (ModuleName is not null) + writer.WriteBytes(ModuleName.GetBytesUnsafe()); + writer.WriteByte(0); + if (ObjectFileName is not null) + writer.WriteBytes(ObjectFileName.GetBytesUnsafe()); + writer.WriteByte(0); + writer.Align(4); + } + + /// + public override string ToString() => ModuleName ?? ObjectFileName ?? "?"; +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs index 35953c208..83daaae3e 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionContribution.cs @@ -1,125 +1,128 @@ -using System.Text; -using AsmResolver.IO; -using AsmResolver.PE.File.Headers; - -namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; - -/// -/// Describes the section in the final executable file that a particular object or module is stored at. -/// -public class SectionContribution : IWritable -{ - /// - /// Gets or sets the index of the section. - /// - public ushort Section - { - get; - set; - } - - /// - /// Gets or sets the offset within the section that this contribution starts at. - /// - public uint Offset - { - get; - set; - } - - /// - /// Gets or sets the size of the section contribution. - /// - public uint Size - { - get; - set; - } - - /// - /// Gets or sets the section flags and permissions associated to this section contribution. - /// - public SectionFlags Characteristics - { - get; - set; - } - - /// - /// Gets or sets the index of the module. - /// - public ushort ModuleIndex - { - get; - set; - } - - /// - /// Gets or sets a cyclic redundancy code that can be used to verify the data section of this contribution. - /// - public uint DataCrc - { - get; - set; - } - - /// - /// Gets or sets a cyclic redundancy code that can be used to verify the relocation section of this contribution. - /// - public uint RelocCrc - { - get; - set; - } - - /// - /// Parses a single section contribution from the provided input stream. - /// - /// The input stream. - /// The parsed section contribution. - public static SectionContribution FromReader(ref BinaryStreamReader reader) - { - var result = new SectionContribution(); - - result.Section = reader.ReadUInt16(); - reader.ReadUInt16(); - result.Offset = reader.ReadUInt32(); - result.Size = reader.ReadUInt32(); - result.Characteristics = (SectionFlags) reader.ReadUInt32(); - result.ModuleIndex = reader.ReadUInt16(); - reader.ReadUInt16(); - result.DataCrc = reader.ReadUInt32(); - result.RelocCrc = reader.ReadUInt32(); - - return result; - } - - /// - public uint GetPhysicalSize() - { - return sizeof(ushort) // Section - + sizeof(ushort) // Padding1 - + sizeof(uint) // Offset - + sizeof(uint) // Size - + sizeof(uint) // Characteristics - + sizeof(ushort) // ModuleIndex - + sizeof(ushort) // Padding2 - + sizeof(uint) // DataCrc - + sizeof(uint) // RelocCrc - ; - } - - /// - public void Write(IBinaryStreamWriter writer) - { - writer.WriteUInt16(Section); - writer.WriteUInt16(0); - writer.WriteUInt32(Offset); - writer.WriteUInt32(Size); - writer.WriteUInt32((uint) Characteristics); - writer.WriteUInt16(ModuleIndex); - writer.WriteUInt16(0); - writer.WriteUInt32(DataCrc); - writer.WriteUInt32(RelocCrc); - } -} +using System.Text; +using AsmResolver.IO; +using AsmResolver.PE.File.Headers; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Describes the section in the final executable file that a particular object or module is stored at. +/// +public class SectionContribution : IWritable +{ + /// + /// The total size in bytes of a single on the disk. + /// + public const int EntrySize = + sizeof(ushort) // Section + + sizeof(ushort) // Padding1 + + sizeof(uint) // Offset + + sizeof(uint) // Size + + sizeof(uint) // Characteristics + + sizeof(ushort) // ModuleIndex + + sizeof(ushort) // Padding2 + + sizeof(uint) // DataCrc + + sizeof(uint) // RelocCrc + ; + + /// + /// Gets or sets the index of the section. + /// + public ushort Section + { + get; + set; + } + + /// + /// Gets or sets the offset within the section that this contribution starts at. + /// + public uint Offset + { + get; + set; + } + + /// + /// Gets or sets the size of the section contribution. + /// + public uint Size + { + get; + set; + } + + /// + /// Gets or sets the section flags and permissions associated to this section contribution. + /// + public SectionFlags Characteristics + { + get; + set; + } + + /// + /// Gets or sets the index of the module. + /// + public ushort ModuleIndex + { + get; + set; + } + + /// + /// Gets or sets a cyclic redundancy code that can be used to verify the data section of this contribution. + /// + public uint DataCrc + { + get; + set; + } + + /// + /// Gets or sets a cyclic redundancy code that can be used to verify the relocation section of this contribution. + /// + public uint RelocCrc + { + get; + set; + } + + /// + /// Parses a single section contribution from the provided input stream. + /// + /// The input stream. + /// The parsed section contribution. + public static SectionContribution FromReader(ref BinaryStreamReader reader) + { + var result = new SectionContribution(); + + result.Section = reader.ReadUInt16(); + reader.ReadUInt16(); + result.Offset = reader.ReadUInt32(); + result.Size = reader.ReadUInt32(); + result.Characteristics = (SectionFlags) reader.ReadUInt32(); + result.ModuleIndex = reader.ReadUInt16(); + reader.ReadUInt16(); + result.DataCrc = reader.ReadUInt32(); + result.RelocCrc = reader.ReadUInt32(); + + return result; + } + + /// + public uint GetPhysicalSize() => EntrySize; + + /// + public void Write(IBinaryStreamWriter writer) + { + writer.WriteUInt16(Section); + writer.WriteUInt16(0); + writer.WriteUInt32(Offset); + writer.WriteUInt32(Size); + writer.WriteUInt32((uint) Characteristics); + writer.WriteUInt16(ModuleIndex); + writer.WriteUInt16(0); + writer.WriteUInt32(DataCrc); + writer.WriteUInt32(RelocCrc); + } +} diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMap.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMap.cs index 6c42bb300..bf43a7e56 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMap.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SectionMap.cs @@ -1,128 +1,131 @@ -using AsmResolver.IO; - -namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; - -/// -/// Represents a single entry in the Section Map sub stream of the DBI stream. -/// -public class SectionMap : IWritable -{ - /// - /// Gets or sets the attributes assigned to this section map. - /// - public SectionMapAttributes Attributes - { - get; - set; - } - - /// - /// Gets or sets the logical overlay number of this section map. - /// - public ushort LogicalOverlayNumber - { - get; - set; - } - - /// - /// Gets or sets the group index into the descriptor array. - /// - public ushort Group - { - get; - set; - } - - /// - /// Gets or sets the frame index. - /// - public ushort Frame - { - get; - set; - } - - /// - /// Gets or sets the byte offset of the segment or group name in string table, or 0xFFFF if no name was assigned. - /// - public ushort SectionName - { - get; - set; - } - - /// - /// Gets or sets the byte offset of the class in the string table, or 0xFFFF if no name was assigned.. - /// - public ushort ClassName - { - get; - set; - } - - /// - /// Gets or sets the byte offset of the logical segment within physical segment. If group is set in flags, this is the offset of the group. - /// - public uint Offset - { - get; - set; - } - - /// - /// Gets or sets the number of bytes that the segment or group consists of. - /// - public uint SectionLength - { - get; - set; - } - - /// - /// Parses a single section map from the provided input stream. - /// - /// The input stream. - /// The parsed section map. - public static SectionMap FromReader(ref BinaryStreamReader reader) - { - return new SectionMap - { - Attributes = (SectionMapAttributes) reader.ReadUInt16(), - LogicalOverlayNumber = reader.ReadUInt16(), - Group = reader.ReadUInt16(), - Frame = reader.ReadUInt16(), - SectionName = reader.ReadUInt16(), - ClassName = reader.ReadUInt16(), - Offset = reader.ReadUInt32(), - SectionLength = reader.ReadUInt32() - }; - } - - /// - public uint GetPhysicalSize() - { - return sizeof(ushort) // Attributes - + sizeof(ushort) // Ovl - + sizeof(ushort) // Group - + sizeof(ushort) // Frame - + sizeof(ushort) // SectionName - + sizeof(ushort) // ClassName - + sizeof(uint) // Offset - + sizeof(uint) // SectionLength - ; - } - - /// - public void Write(IBinaryStreamWriter writer) - { - writer.WriteUInt16((ushort) Attributes); - writer.WriteUInt16(LogicalOverlayNumber); - writer.WriteUInt16(Group); - writer.WriteUInt16(Frame); - writer.WriteUInt16(SectionName); - writer.WriteUInt16(ClassName); - writer.WriteUInt32(Offset); - writer.WriteUInt32(SectionLength); - } -} +using AsmResolver.IO; + +namespace AsmResolver.Symbols.Pdb.Metadata.Dbi; + +/// +/// Represents a single entry in the Section Map sub stream of the DBI stream. +/// +public class SectionMap : IWritable +{ + /// + /// The total size in bytes of a single on the disk. + /// + public const int EntrySize = + sizeof(ushort) // Attributes + + sizeof(ushort) // Ovl + + sizeof(ushort) // Group + + sizeof(ushort) // Frame + + sizeof(ushort) // SectionName + + sizeof(ushort) // ClassName + + sizeof(uint) // Offset + + sizeof(uint) // SectionLength + ; + + /// + /// Gets or sets the attributes assigned to this section map. + /// + public SectionMapAttributes Attributes + { + get; + set; + } + + /// + /// Gets or sets the logical overlay number of this section map. + /// + public ushort LogicalOverlayNumber + { + get; + set; + } + + /// + /// Gets or sets the group index into the descriptor array. + /// + public ushort Group + { + get; + set; + } + + /// + /// Gets or sets the frame index. + /// + public ushort Frame + { + get; + set; + } + + /// + /// Gets or sets the byte offset of the segment or group name in string table, or 0xFFFF if no name was assigned. + /// + public ushort SectionName + { + get; + set; + } + + /// + /// Gets or sets the byte offset of the class in the string table, or 0xFFFF if no name was assigned.. + /// + public ushort ClassName + { + get; + set; + } + + /// + /// Gets or sets the byte offset of the logical segment within physical segment. If group is set in flags, this is the offset of the group. + /// + public uint Offset + { + get; + set; + } + + /// + /// Gets or sets the number of bytes that the segment or group consists of. + /// + public uint SectionLength + { + get; + set; + } + + /// + /// Parses a single section map from the provided input stream. + /// + /// The input stream. + /// The parsed section map. + public static SectionMap FromReader(ref BinaryStreamReader reader) + { + return new SectionMap + { + Attributes = (SectionMapAttributes) reader.ReadUInt16(), + LogicalOverlayNumber = reader.ReadUInt16(), + Group = reader.ReadUInt16(), + Frame = reader.ReadUInt16(), + SectionName = reader.ReadUInt16(), + ClassName = reader.ReadUInt16(), + Offset = reader.ReadUInt32(), + SectionLength = reader.ReadUInt32() + }; + } + + /// + public uint GetPhysicalSize() => EntrySize; + + /// + public void Write(IBinaryStreamWriter writer) + { + writer.WriteUInt16((ushort) Attributes); + writer.WriteUInt16(LogicalOverlayNumber); + writer.WriteUInt16(Group); + writer.WriteUInt16(Frame); + writer.WriteUInt16(SectionName); + writer.WriteUInt16(ClassName); + writer.WriteUInt32(Offset); + writer.WriteUInt32(SectionLength); + } +} diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs index 179173850..35a2efb61 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -1,172 +1,196 @@ -using System; -using System.Linq; -using AsmResolver.PE.File.Headers; -using AsmResolver.Symbols.Pdb.Metadata.Dbi; -using AsmResolver.Symbols.Pdb.Msf; -using Xunit; - -namespace AsmResolver.Symbols.Pdb.Tests.Metadata.Dbi; - -public class DbiStreamTest -{ - [Fact] - public void ReadHeader() - { - var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); - var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); - - Assert.Equal(1u, dbiStream.Age); - Assert.Equal(DbiAttributes.None, dbiStream.Attributes); - Assert.Equal(MachineType.I386, dbiStream.Machine); - } - - [Fact] - public void ReadModuleNames() - { - var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); - var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); - - Assert.Equal(new[] - { - "* CIL *", - "C:\\Users\\Admin\\source\\repos\\AsmResolver\\test\\TestBinaries\\Native\\SimpleDll\\Release\\dllmain.obj", - "C:\\Users\\Admin\\source\\repos\\AsmResolver\\test\\TestBinaries\\Native\\SimpleDll\\Release\\pch.obj", - "* Linker Generated Manifest RES *", - "Import:KERNEL32.dll", - "KERNEL32.dll", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\sehprolg4.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_cookie.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_report.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_support.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\guard_support.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\loadcfg.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\dyn_tls_init.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\ucrt_detection.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\cpu_disp.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\chandler4gs.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\secchk.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\argv_mode.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\default_local_stdio_options.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\tncleanup.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\dll_dllmain.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\initializers.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\utility.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\ucrt_stubs.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\utility_desktop.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\initsect.obj", - "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\x86_exception_filter.obj", - "VCRUNTIME140.dll", - "Import:VCRUNTIME140.dll", - "Import:api-ms-win-crt-runtime-l1-1-0.dll", - "api-ms-win-crt-runtime-l1-1-0.dll", - "* Linker *", - }, dbiStream.Modules.Select(m => m.ModuleName?.Value)); - } - - [Fact] - public void ReadSectionContributions() - { - var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); - var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); - - Assert.Equal(new (ushort, uint)[] - { - (1, 1669053862), (16, 2162654757), (20, 1635644926), (20, 3159649454), (20, 1649652954), (20, 3877379438), - (20, 4262788820), (20, 199934614), (8, 4235719287), (8, 1374843914), (9, 4241735292), (9, 2170796787), - (19, 1300950661), (19, 3968158929), (18, 3928463356), (18, 3928463356), (18, 2109213706), (22, 1457516325), - (22, 3939645857), (22, 1393694582), (22, 546064581), (22, 1976627334), (22, 513172946), (22, 25744891), - (22, 1989765812), (22, 2066266302), (22, 3810887196), (22, 206965504), (22, 647717352), (22, 3911072265), - (22, 3290064241), (12, 3928463356), (24, 2717331243), (24, 3687876222), (25, 2318145338), (25, 2318145338), - (6, 542071654), (15, 1810708069), (10, 3974941622), (14, 1150179208), (17, 2709606169), (13, 2361171624), - (28, 0), (28, 0), (28, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), - (23, 3467414241), (23, 4079273803), (26, 1282639619), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), - (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (28, 0), (28, 0), (28, 0), (27, 0), (29, 0), (29, 0), - (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (10, 2556510175), (21, 2556510175), - (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), - (21, 2556510175), (20, 2556510175), (8, 4117779887), (31, 0), (11, 525614319), (31, 0), (31, 0), (31, 0), - (31, 0), (31, 0), (25, 2556510175), (25, 2556510175), (25, 2556510175), (25, 2556510175), (20, 3906165615), - (20, 1185345766), (20, 407658226), (22, 2869884627), (27, 0), (30, 0), (5, 0), (27, 0), (4, 0), (4, 0), - (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (28, 0), (28, 0), (28, 0), - (27, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (28, 0), (28, 0), - (28, 0), (27, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (4, 0), - (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (7, 4096381681), - (22, 454268333), (14, 1927129959), (23, 1927129959), (20, 0), (8, 0), (19, 0), (18, 0), (18, 0), (22, 0), - (24, 0), (10, 0), (14, 0), (2, 0), (31, 0), (3, 0), (3, 0) - }, dbiStream.SectionContributions.Select(x => (x.ModuleIndex, x.DataCrc))); - } - - [Fact] - public void ReadSectionMap() - { - var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); - var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); - - Assert.Equal(new (ushort, ushort, ushort, ushort, ushort, ushort, uint, uint)[] - { - (0x010d, 0x0000, 0x0000, 0x0001, 0xffff, 0xffff, 0x00000000, 0x00000ce8), - (0x0109, 0x0000, 0x0000, 0x0002, 0xffff, 0xffff, 0x00000000, 0x00000834), - (0x010b, 0x0000, 0x0000, 0x0003, 0xffff, 0xffff, 0x00000000, 0x00000394), - (0x0109, 0x0000, 0x0000, 0x0004, 0xffff, 0xffff, 0x00000000, 0x000000f8), - (0x0109, 0x0000, 0x0000, 0x0005, 0xffff, 0xffff, 0x00000000, 0x0000013c), - (0x0208, 0x0000, 0x0000, 0x0000, 0xffff, 0xffff, 0x00000000, 0xffffffff), - }, - dbiStream.SectionMaps.Select(m => ((ushort) - m.Attributes, m.LogicalOverlayNumber, m.Group, m.Frame, - m.SectionName, m.ClassName, m.Offset, m.SectionLength))); - } - - [Fact] - public void ReadSourceFiles() - { - var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); - var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); - - string[][] firstThreeActualFileLists = dbiStream.SourceFiles - .Take(3) - .Select(x => x - .Select(y => y.ToString()) - .ToArray() - ).ToArray(); - - Assert.Equal(new[] - { - Array.Empty(), - new[] - { - @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\pch.h", - @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\dllmain.cpp", - @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\Release\SimpleDll.pch", - }, - new[] - { - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winuser.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\basetsd.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winbase.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\stralign.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\guiddef.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\winerror.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_wstring.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\processthreadsapi.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winnt.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\ctype.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\string.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_memory.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\memoryapi.h", - @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_memcpy_s.h", - @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\Release\SimpleDll.pch", - } - }, - firstThreeActualFileLists); - } - - [Fact] - public void ReadExtraDebugIndices() - { - var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); - var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); - Assert.Equal(new ushort[] - { - 0x7, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xB, 0xFFFF, 0xFFFF, 0xFFFF, 0xD, 0xFFFF - }, dbiStream.ExtraStreamIndices); - } -} +using System; +using System.IO; +using System.Linq; +using AsmResolver.IO; +using AsmResolver.PE.File.Headers; +using AsmResolver.Symbols.Pdb.Metadata.Dbi; +using AsmResolver.Symbols.Pdb.Msf; +using Xunit; + +namespace AsmResolver.Symbols.Pdb.Tests.Metadata.Dbi; + +public class DbiStreamTest +{ + private DbiStream GetDbiStream(bool rebuild) + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var dbiStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); + + if (rebuild) + { + using var stream = new MemoryStream(); + dbiStream.Write(new BinaryStreamWriter(stream)); + dbiStream = DbiStream.FromReader(ByteArrayDataSource.CreateReader(stream.ToArray())); + } + + return dbiStream; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Header(bool rebuild) + { + var dbiStream = GetDbiStream(rebuild); + + Assert.Equal(1u, dbiStream.Age); + Assert.Equal(DbiAttributes.None, dbiStream.Attributes); + Assert.Equal(MachineType.I386, dbiStream.Machine); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ModuleNames(bool rebuild) + { + var dbiStream = GetDbiStream(rebuild); + + Assert.Equal(new[] + { + "* CIL *", + "C:\\Users\\Admin\\source\\repos\\AsmResolver\\test\\TestBinaries\\Native\\SimpleDll\\Release\\dllmain.obj", + "C:\\Users\\Admin\\source\\repos\\AsmResolver\\test\\TestBinaries\\Native\\SimpleDll\\Release\\pch.obj", + "* Linker Generated Manifest RES *", + "Import:KERNEL32.dll", + "KERNEL32.dll", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\sehprolg4.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_cookie.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_report.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\gs_support.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\guard_support.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\loadcfg.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\dyn_tls_init.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\ucrt_detection.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\cpu_disp.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\chandler4gs.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\secchk.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\argv_mode.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\default_local_stdio_options.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\tncleanup.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\dll_dllmain.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\initializers.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\utility.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\ucrt_stubs.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\utility_desktop.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\initsect.obj", + "D:\\a\\_work\\1\\s\\Intermediate\\vctools\\msvcrt.nativeproj_110336922\\objr\\x86\\x86_exception_filter.obj", + "VCRUNTIME140.dll", + "Import:VCRUNTIME140.dll", + "Import:api-ms-win-crt-runtime-l1-1-0.dll", + "api-ms-win-crt-runtime-l1-1-0.dll", + "* Linker *", + }, dbiStream.Modules.Select(m => m.ModuleName?.Value)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SectionContributions(bool rebuild) + { + var dbiStream = GetDbiStream(rebuild); + + Assert.Equal(new (ushort, uint)[] + { + (1, 1669053862), (16, 2162654757), (20, 1635644926), (20, 3159649454), (20, 1649652954), (20, 3877379438), + (20, 4262788820), (20, 199934614), (8, 4235719287), (8, 1374843914), (9, 4241735292), (9, 2170796787), + (19, 1300950661), (19, 3968158929), (18, 3928463356), (18, 3928463356), (18, 2109213706), (22, 1457516325), + (22, 3939645857), (22, 1393694582), (22, 546064581), (22, 1976627334), (22, 513172946), (22, 25744891), + (22, 1989765812), (22, 2066266302), (22, 3810887196), (22, 206965504), (22, 647717352), (22, 3911072265), + (22, 3290064241), (12, 3928463356), (24, 2717331243), (24, 3687876222), (25, 2318145338), (25, 2318145338), + (6, 542071654), (15, 1810708069), (10, 3974941622), (14, 1150179208), (17, 2709606169), (13, 2361171624), + (28, 0), (28, 0), (28, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), + (23, 3467414241), (23, 4079273803), (26, 1282639619), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), + (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (28, 0), (28, 0), (28, 0), (27, 0), (29, 0), (29, 0), + (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (10, 2556510175), (21, 2556510175), + (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), (21, 2556510175), + (21, 2556510175), (20, 2556510175), (8, 4117779887), (31, 0), (11, 525614319), (31, 0), (31, 0), (31, 0), + (31, 0), (31, 0), (25, 2556510175), (25, 2556510175), (25, 2556510175), (25, 2556510175), (20, 3906165615), + (20, 1185345766), (20, 407658226), (22, 2869884627), (27, 0), (30, 0), (5, 0), (27, 0), (4, 0), (4, 0), + (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (28, 0), (28, 0), (28, 0), + (27, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (28, 0), (28, 0), + (28, 0), (27, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (29, 0), (30, 0), (4, 0), + (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (4, 0), (5, 0), (7, 4096381681), + (22, 454268333), (14, 1927129959), (23, 1927129959), (20, 0), (8, 0), (19, 0), (18, 0), (18, 0), (22, 0), + (24, 0), (10, 0), (14, 0), (2, 0), (31, 0), (3, 0), (3, 0) + }, dbiStream.SectionContributions.Select(x => (x.ModuleIndex, x.DataCrc))); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SectionMaps(bool rebuild) + { + var dbiStream = GetDbiStream(rebuild); + + Assert.Equal(new (ushort, ushort, ushort, ushort, ushort, ushort, uint, uint)[] + { + (0x010d, 0x0000, 0x0000, 0x0001, 0xffff, 0xffff, 0x00000000, 0x00000ce8), + (0x0109, 0x0000, 0x0000, 0x0002, 0xffff, 0xffff, 0x00000000, 0x00000834), + (0x010b, 0x0000, 0x0000, 0x0003, 0xffff, 0xffff, 0x00000000, 0x00000394), + (0x0109, 0x0000, 0x0000, 0x0004, 0xffff, 0xffff, 0x00000000, 0x000000f8), + (0x0109, 0x0000, 0x0000, 0x0005, 0xffff, 0xffff, 0x00000000, 0x0000013c), + (0x0208, 0x0000, 0x0000, 0x0000, 0xffff, 0xffff, 0x00000000, 0xffffffff), + }, + dbiStream.SectionMaps.Select(m => ((ushort) + m.Attributes, m.LogicalOverlayNumber, m.Group, m.Frame, + m.SectionName, m.ClassName, m.Offset, m.SectionLength))); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SourceFiles(bool rebuild) + { + var dbiStream = GetDbiStream(rebuild); + + string[][] firstThreeActualFileLists = dbiStream.SourceFiles + .Take(3) + .Select(x => x + .Select(y => y.ToString()) + .ToArray() + ).ToArray(); + + Assert.Equal(new[] + { + Array.Empty(), + new[] + { + @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\pch.h", + @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\dllmain.cpp", + @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\Release\SimpleDll.pch", + }, + new[] + { + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winuser.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\basetsd.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winbase.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\stralign.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\guiddef.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\shared\winerror.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_wstring.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\processthreadsapi.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\winnt.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\ctype.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\string.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_memory.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\um\memoryapi.h", + @"C:\Program Files (x86)\Windows Kits\10\Include\10.0.19041.0\ucrt\corecrt_memcpy_s.h", + @"C:\Users\Admin\source\repos\AsmResolver\test\TestBinaries\Native\SimpleDll\Release\SimpleDll.pch", + } + }, + firstThreeActualFileLists); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ExtraDebugIndices(bool rebuild) + { + var dbiStream = GetDbiStream(rebuild); + + Assert.Equal(new ushort[] + { + 0x7, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xB, 0xFFFF, 0xFFFF, 0xFFFF, 0xD, 0xFFFF + }, dbiStream.ExtraStreamIndices); + } +} From d510ec9f207d664096efe2644fa48cdc540ab2d5 Mon Sep 17 00:00:00 2001 From: Washi Date: Fri, 24 Jun 2022 18:40:47 +0200 Subject: [PATCH 13/16] Add BinaryStreamReader.ReadUtf8String --- .../Metadata/Strings/SerializedStringsStream.cs | 7 +------ .../Metadata/Dbi/ModuleDescriptor.cs | 4 ++-- .../Metadata/Dbi/SerializedDbiStream.cs | 2 +- .../Metadata/Info/SerializedInfoStream.cs | 7 +------ src/AsmResolver/IO/BinaryStreamReader.cs | 12 ++++++++++++ 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/AsmResolver.PE/DotNet/Metadata/Strings/SerializedStringsStream.cs b/src/AsmResolver.PE/DotNet/Metadata/Strings/SerializedStringsStream.cs index 78532dfb1..293562f36 100644 --- a/src/AsmResolver.PE/DotNet/Metadata/Strings/SerializedStringsStream.cs +++ b/src/AsmResolver.PE/DotNet/Metadata/Strings/SerializedStringsStream.cs @@ -65,12 +65,7 @@ public SerializedStringsStream(string name, in BinaryStreamReader reader) if (!_cachedStrings.TryGetValue(index, out var value) && index < _reader.Length) { var stringsReader = _reader.ForkRelative(index); - byte[] rawData = stringsReader.ReadBytesUntil(0, false); - - value = rawData.Length != 0 - ? new Utf8String(rawData) - : Utf8String.Empty; - + value = stringsReader.ReadUtf8String(); _cachedStrings[index] = value; } diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs index 5fee61c3e..174ec2dab 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/ModuleDescriptor.cs @@ -151,8 +151,8 @@ public static ModuleDescriptor FromReader(ref BinaryStreamReader reader) reader.ReadUInt32(); result.SourceFileNameIndex = reader.ReadUInt32(); result.PdbFilePathNameIndex = reader.ReadUInt32(); - result.ModuleName = new Utf8String(reader.ReadBytesUntil(0, false)); - result.ObjectFileName = new Utf8String(reader.ReadBytesUntil(0, false)); + result.ModuleName = reader.ReadUtf8String(); + result.ObjectFileName = reader.ReadUtf8String(); reader.Align(4); return result; diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index 4c6fd2632..172f44a5a 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -166,7 +166,7 @@ protected override IList GetSourceFiles() { uint nameOffset = reader.ReadUInt32(); var nameReader = stringReaderBuffer.ForkRelative(nameOffset); - files.Add(new Utf8String(nameReader.ReadBytesUntil(0, false))); + files.Add(nameReader.ReadUtf8String()); } result.Add(files); diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs index 8ce5c3b21..1aebc173a 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs @@ -42,12 +42,7 @@ protected override IDictionary GetStreamIndices() var result = PdbHashTable.FromReader(ref hashTableReader, (key, value) => { var stringReader = stringsReader.ForkRelative(key); - byte[] rawData = stringReader.ReadBytesUntil(0, false); - - var keyString = rawData.Length != 0 - ? new Utf8String(rawData) - : Utf8String.Empty; - + var keyString = stringReader.ReadUtf8String(); return (keyString, (int) value); }); diff --git a/src/AsmResolver/IO/BinaryStreamReader.cs b/src/AsmResolver/IO/BinaryStreamReader.cs index 7a48e29be..d339ceab3 100644 --- a/src/AsmResolver/IO/BinaryStreamReader.cs +++ b/src/AsmResolver/IO/BinaryStreamReader.cs @@ -395,6 +395,18 @@ public string ReadUnicodeString() return builder.ToString(); } + /// + /// Reads a null-terminated UTF-8 string from the input stream. + /// + /// The read UTF-8 string, excluding the null terminator. + public Utf8String ReadUtf8String() + { + byte[] data = ReadBytesUntil(0, false); + return data.Length != 0 + ? new Utf8String(data) + : Utf8String.Empty; + } + /// /// Reads either a 32-bit or a 64-bit number from the input stream. /// From e4ef4c11a7de2ba25e9e56dcf70528050cecc594 Mon Sep 17 00:00:00 2001 From: Washi Date: Fri, 24 Jun 2022 18:54:17 +0200 Subject: [PATCH 14/16] Explode DbiStream.BuildNumber into major, minor and newfileformat version properties. --- .../Metadata/Dbi/DbiStream.cs | 28 +++++++++++++++++++ .../Metadata/Dbi/SerializedDbiStream.cs | 4 +++ .../Metadata/Dbi/DbiStreamTest.cs | 3 ++ 3 files changed, 35 insertions(+) diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs index e9180fb4d..247bd21ac 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/DbiStream.cs @@ -32,6 +32,7 @@ public DbiStream() { _typeServerMapStream = new LazyVariable(GetTypeServerMapStream); _ecStream = new LazyVariable(GetECStream); + IsNewVersionFormat = true; } /// @@ -85,6 +86,33 @@ public ushort BuildNumber set; } + /// + /// Gets or sets a value indicating that the DBI stream is using the new file format (NewDBI). + /// + public bool IsNewVersionFormat + { + get => (BuildNumber & 0x8000) != 0; + set => BuildNumber = (ushort) ((BuildNumber & ~0x8000) | (value ? 0x8000 : 0)); + } + + /// + /// Gets or sets the major version of the toolchain that was used to build the program. + /// + public byte BuildMajorVersion + { + get => (byte) ((BuildNumber >> 8) & 0x7F); + set => BuildNumber = (ushort) ((BuildNumber & ~0x7F00) | (value << 8)); + } + + /// + /// Gets or sets the minor version of the toolchain that was used to build the program. + /// + public byte BuildMinorVersion + { + get => (byte) (BuildNumber & 0xFF); + set => BuildNumber = (ushort) ((BuildNumber & ~0x00FF) | value); + } + /// /// Gets or sets the MSF stream index of the Public Symbol Stream. /// diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs index 172f44a5a..962f02fc7 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Dbi/SerializedDbiStream.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using AsmResolver.IO; using AsmResolver.PE.File.Headers; @@ -28,6 +29,9 @@ public SerializedDbiStream(BinaryStreamReader reader) Age = reader.ReadUInt32(); GlobalStreamIndex = reader.ReadUInt16(); BuildNumber = reader.ReadUInt16(); + if (!IsNewVersionFormat) + throw new NotSupportedException("The DBI stream uses the legacy file format, which is not supported."); + PublicStreamIndex = reader.ReadUInt16(); PdbDllVersion = reader.ReadUInt16(); SymbolRecordStreamIndex = reader.ReadUInt16(); diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs index 35a2efb61..19e2c5035 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -36,6 +36,9 @@ public void Header(bool rebuild) Assert.Equal(1u, dbiStream.Age); Assert.Equal(DbiAttributes.None, dbiStream.Attributes); Assert.Equal(MachineType.I386, dbiStream.Machine); + Assert.Equal(14, dbiStream.BuildMajorVersion); + Assert.Equal(29, dbiStream.BuildMinorVersion); + Assert.True(dbiStream.IsNewVersionFormat); } [Theory] From c7039c55068ccf0cb16781da316dbde50a22dac8 Mon Sep 17 00:00:00 2001 From: Washi Date: Sat, 25 Jun 2022 16:27:27 +0200 Subject: [PATCH 15/16] BUGFIX: Include pdb hash table and feature codes in size computation of InfoStream. --- .../Metadata/Info/InfoStream.cs | 45 +++++-- .../Metadata/Info/SerializedInfoStream.cs | 2 +- .../Metadata/PdbHashTable.cs | 115 +++++++++++++----- .../Metadata/Dbi/DbiStreamTest.cs | 14 +++ .../Metadata/Info/InfoStreamTest.cs | 48 +++++++- .../Msf/MsfFileTest.cs | 1 - 6 files changed, 181 insertions(+), 44 deletions(-) diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs index 00fede54c..5dd16b073 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs @@ -16,6 +16,14 @@ public class InfoStream : SegmentBase /// public const int StreamIndex = 1; + private const int HeaderSize = + sizeof(InfoStreamVersion) // Version + + sizeof(uint) // Signature + + sizeof(uint) // Aage + + 16 //UniqueId + + sizeof(uint) // NameBufferSize + ; + private IDictionary? _streamIndices; private IList? _features; @@ -115,11 +123,22 @@ public IList Features /// public override uint GetPhysicalSize() { - return sizeof(uint) // Version - + sizeof(uint) // Signature - + sizeof(uint) // Aage - + 16 // UniqueId - ; + uint totalSize = HeaderSize; + + // Name buffer + foreach (var entry in StreamIndices) + totalSize += (uint) entry.Key.ByteCount + 1u; + + // Stream indices hash table. + totalSize += StreamIndices.GetPdbHashTableSize(ComputeStringHash); + + // Last NI + totalSize += sizeof(uint); + + // Feature codes. + totalSize += (uint) Features.Count * sizeof(PdbFeature); + + return totalSize; } /// @@ -148,12 +167,8 @@ public override void Write(IBinaryStreamWriter writer) writer.WriteBytes(nameBuffer.ToArray()); // Write the hash table. - // Note: The hash of a single entry is **deliberately** truncated to a 16 bit number. This is because - // the reference implementation of the name table returns a number of type HASH, which is a typedef - // for "unsigned short". If we don't do this, this will result in wrong buckets being filled in the - // hash table, and thus the serialization would fail. See NMTNI::hash() in Microsoft/microsoft-pdb. StreamIndices.WriteAsPdbHashTable(writer, - str => (ushort) PdbHash.ComputeV1(str), + ComputeStringHash, (key, value) => (stringOffsets[key], (uint) value)); // last NI, safe to put always zero. @@ -163,4 +178,14 @@ public override void Write(IBinaryStreamWriter writer) foreach (var feature in Features) writer.WriteUInt32((uint) feature); } + + private static uint ComputeStringHash(Utf8String str) + { + // Note: The hash of a single entry is **deliberately** truncated to a 16 bit number. This is because + // the reference implementation of the name table returns a number of type HASH, which is a typedef + // for "unsigned short". If we don't do this, this will result in wrong buckets being filled in the + // hash table, and thus the serialization would fail. See NMTNI::hash() in Microsoft/microsoft-pdb. + + return (ushort) PdbHash.ComputeV1(str); + } } diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs index 1aebc173a..f448ce67f 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/SerializedInfoStream.cs @@ -46,7 +46,7 @@ protected override IDictionary GetStreamIndices() return (keyString, (int) value); }); - uint lastNi = hashTableReader.ReadUInt32(); // Unused. + hashTableReader.ReadUInt32(); // lastNi (unused). _featureOffset = hashTableReader.Offset; return result; diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs b/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs index 7125fed86..fb58f25be 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/PdbHashTable.cs @@ -26,8 +26,8 @@ public static Dictionary FromReader( Func mapper) where TKey : notnull { - uint size = reader.ReadUInt32(); - uint capacity = reader.ReadUInt32(); + uint count = reader.ReadUInt32(); + reader.ReadUInt32(); // Capacity uint presentWordCount = reader.ReadUInt32(); reader.RelativeOffset += presentWordCount * sizeof(uint); @@ -35,8 +35,8 @@ public static Dictionary FromReader( uint deletedWordCount = reader.ReadUInt32(); reader.RelativeOffset += deletedWordCount * sizeof(uint); - var result = new Dictionary(); - for (int i = 0; i < size; i++) + var result = new Dictionary((int) count); + for (int i = 0; i < count; i++) { (uint rawKey, uint rawValue) = (reader.ReadUInt32(), reader.ReadUInt32()); var (key, value) = mapper(rawKey, rawValue); @@ -47,7 +47,31 @@ public static Dictionary FromReader( } /// - /// Serializes a dictionary to a PDB hash table. + /// Computes the number of bytes required to store the provided dictionary as a PDB hash table. + /// + /// The dictionary to serialize. + /// A function that computes the hash code for a single key within the dictionary. + /// The type of keys in the input dictionary. + /// The type of values in the input dictionary. + /// The number of bytes required. + public static uint GetPdbHashTableSize( + this IDictionary dictionary, + Func hasher) + where TKey : notnull + { + var info = dictionary.ToPdbHashTable(hasher, null); + + return sizeof(uint) // Count + + sizeof(uint) // Capacity + + sizeof(uint) // Present bitvector word count + + info.PresentWordCount * sizeof(uint) // Present bitvector words + + sizeof(uint) // Deleted bitvector word count (== 0) + + (sizeof(uint) + sizeof(uint)) * (uint) dictionary.Count + ; + } + + /// + /// Serializes a dictionary to a PDB hash table to an output stream. /// /// The dictionary to serialize. /// The output stream to write to. @@ -84,12 +108,12 @@ public static void WriteAsPdbHashTable( writer.WriteUInt32(0); // Write all buckets. - for (int i = 0; i < hashTable.Keys.Length; i++) + for (int i = 0; i < hashTable.Keys!.Length; i++) { if (hashTable.Present.Get(i)) { - writer.WriteUInt32(hashTable.Keys[i]); - writer.WriteUInt32(hashTable.Values[i]); + writer.WriteUInt32(hashTable.Keys![i]); + writer.WriteUInt32(hashTable.Values![i]); } } } @@ -97,56 +121,91 @@ public static void WriteAsPdbHashTable( private static HashTableInfo ToPdbHashTable( this IDictionary dictionary, Func hasher, - Func mapper) + Func? mapper) where TKey : notnull { - // "Simulate" adding all items to the hash table, effectively calculating the capacity of the map. - // TODO: This can probably be calculated with a single formula instead. - uint capacity = 1; - for (int i = 0; i <= dictionary.Count; i++) + uint capacity = ComputeRequiredCapacity(dictionary.Count); + + // Avoid allocating buckets if we actually don't need to (e.g. if we're simply measuring the total size). + uint[]? keys; + uint[]? values; + + if (mapper is null) { - // Reference implementation allows only 67% of the capacity to be used. - uint maxLoad = capacity * 2 / 3 + 1; - if (i >= maxLoad) - capacity = 2 * maxLoad; + keys = null; + values = null; + } + else + { + keys = new uint[capacity]; + values = new uint[capacity]; } - // Define buckets. - uint[] keys = new uint[capacity]; - uint[] values = new uint[capacity]; var present = new BitArray((int) capacity, false); // Fill in buckets. foreach (var item in dictionary) { + // Find empty bucket to place key-value pair in. uint hash = hasher(item.Key); - (uint key, uint value) = mapper(item.Key, item.Value); - uint index = hash % capacity; while (present.Get((int) index)) index = (index + 1) % capacity; - keys[index] = key; - values[index] = value; + // Mark bucket as used. present.Set((int) index, true); + + // Store key-value pair. + if (mapper is not null) + { + (uint key, uint value) = mapper(item.Key, item.Value); + keys![index] = key; + values![index] = value; + } + } + + // Determine final word count in present bit vector. + uint wordCount = (capacity + sizeof(uint) - 1) / sizeof(uint); + uint[] words = new uint[wordCount]; + present.CopyTo(words, 0); + while (wordCount > 0 && words[wordCount - 1] == 0) + wordCount--; + + return new HashTableInfo(capacity, keys, values, present, wordCount); + } + + private static uint ComputeRequiredCapacity(int totalItemCount) + { + // "Simulate" adding all items to the hash table, effectively calculating the capacity of the map. + // TODO: This can probably be calculated with a single formula instead. + + uint capacity = 1; + for (int i = 0; i <= totalItemCount; i++) + { + // Reference implementation allows only 67% of the capacity to be used. + uint maxLoad = capacity * 2 / 3 + 1; + if (i >= maxLoad) + capacity = 2 * maxLoad; } - return new HashTableInfo(capacity, keys, values, present); + return capacity; } private readonly struct HashTableInfo { public readonly uint Capacity; - public readonly uint[] Keys; - public readonly uint[] Values; + public readonly uint[]? Keys; + public readonly uint[]? Values; public readonly BitArray Present; + public readonly uint PresentWordCount; - public HashTableInfo(uint capacity, uint[] keys, uint[] values, BitArray present) + public HashTableInfo(uint capacity, uint[]? keys, uint[]? values, BitArray present, uint presentWordCount) { Capacity = capacity; Keys = keys; Values = values; Present = present; + PresentWordCount = presentWordCount; } } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs index 19e2c5035..889913627 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Dbi/DbiStreamTest.cs @@ -196,4 +196,18 @@ public void ExtraDebugIndices(bool rebuild) 0x7, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xB, 0xFFFF, 0xFFFF, 0xFFFF, 0xD, 0xFFFF }, dbiStream.ExtraStreamIndices); } + + [Fact] + public void SizeCalculation() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var infoStream = DbiStream.FromReader(file.Streams[DbiStream.StreamIndex].CreateReader()); + + uint calculatedSize = infoStream.GetPhysicalSize(); + + using var stream = new MemoryStream(); + infoStream.Write(new BinaryStreamWriter(stream)); + + Assert.Equal(stream.Length, calculatedSize); + } } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs index ff78a7b81..03a19f01b 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Metadata/Info/InfoStreamTest.cs @@ -10,13 +10,11 @@ namespace AsmResolver.Symbols.Pdb.Tests.Metadata.Info; public class InfoStreamTest { - [Theory] - [InlineData(false)] - [InlineData(true)] - public void ReadWrite(bool rebuild) + private static InfoStream GetInfoStream(bool rebuild) { var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); var infoStream = InfoStream.FromReader(file.Streams[InfoStream.StreamIndex].CreateReader()); + if (rebuild) { using var stream = new MemoryStream(); @@ -24,9 +22,28 @@ public void ReadWrite(bool rebuild) infoStream = InfoStream.FromReader(ByteArrayDataSource.CreateReader(stream.ToArray())); } + return infoStream; + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Header(bool rebuild) + { + var infoStream = GetInfoStream(rebuild); + Assert.Equal(InfoStreamVersion.VC70, infoStream.Version); Assert.Equal(1u, infoStream.Age); Assert.Equal(Guid.Parse("205dc366-d8f8-4175-8e06-26dd76722df5"), infoStream.UniqueId); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void NameTable(bool rebuild) + { + var infoStream = GetInfoStream(rebuild); + Assert.Equal(new Dictionary { ["/UDTSRCLINEUNDONE"] = 48, @@ -35,6 +52,29 @@ public void ReadWrite(bool rebuild) ["/TMCache"] = 6, ["/names"] = 12 }, infoStream.StreamIndices); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void FeatureCodes(bool rebuild) + { + var infoStream = GetInfoStream(rebuild); + Assert.Equal(new[] {PdbFeature.VC140}, infoStream.Features); } + + [Fact] + public void SizeCalculation() + { + var file = MsfFile.FromBytes(Properties.Resources.SimpleDllPdb); + var infoStream = InfoStream.FromReader(file.Streams[InfoStream.StreamIndex].CreateReader()); + + uint calculatedSize = infoStream.GetPhysicalSize(); + + using var stream = new MemoryStream(); + infoStream.Write(new BinaryStreamWriter(stream)); + + Assert.Equal(stream.Length, calculatedSize); + } } diff --git a/test/AsmResolver.Symbols.Pdb.Tests/Msf/MsfFileTest.cs b/test/AsmResolver.Symbols.Pdb.Tests/Msf/MsfFileTest.cs index e7c589c70..b17cd4b36 100644 --- a/test/AsmResolver.Symbols.Pdb.Tests/Msf/MsfFileTest.cs +++ b/test/AsmResolver.Symbols.Pdb.Tests/Msf/MsfFileTest.cs @@ -1,6 +1,5 @@ using System.IO; using System.Linq; -using AsmResolver.IO; using AsmResolver.Symbols.Pdb.Msf; using Xunit; From e8717182094691221d1b65fa1d71900c54077942 Mon Sep 17 00:00:00 2001 From: Washi Date: Sat, 25 Jun 2022 16:29:48 +0200 Subject: [PATCH 16/16] Use for loop to avoid heap allocated enumerator. --- src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs index 5dd16b073..eae02174c 100644 --- a/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs +++ b/src/AsmResolver.Symbols.Pdb/Metadata/Info/InfoStream.cs @@ -175,8 +175,9 @@ public override void Write(IBinaryStreamWriter writer) writer.WriteUInt32(0); // Write feature codes. - foreach (var feature in Features) - writer.WriteUInt32((uint) feature); + var features = Features; + for (int i = 0; i < features.Count; i++) + writer.WriteUInt32((uint) features[i]); } private static uint ComputeStringHash(Utf8String str)