Report error for native PDB limit

src/Compilers/CSharp/Test/Emit2/PDB/PDBTests.cs

@@ -5,7 +5,6 @@
 #nullable disable
 
 using System;
-using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Reflection;
@@ -19,9 +18,11 @@
 using Microsoft.CodeAnalysis.Emit;
 using Microsoft.CodeAnalysis.Test.Utilities;
 using Microsoft.CodeAnalysis.Text;
+using Microsoft.DiaSymReader;
 using Roslyn.Test.PdbUtilities;
 using Roslyn.Test.Utilities;
 using Roslyn.Test.Utilities.TestGenerators;
+using Roslyn.Utilities;
 using Xunit;
 
 namespace Microsoft.CodeAnalysis.CSharp.UnitTests.PDB
@@ -367,6 +368,87 @@ public void SymWriterErrors4()
             Assert.False(result.Success);
         }
 
+        /// <summary>
+        /// Verifies the constant <c>SymUnmanagedWriterImpl.CustomMetadataByteLimit</c> against the external sym writer library we depend on.
+        /// </summary>
+        [ConditionalTheory(typeof(WindowsOnly), Reason = ConditionalSkipReason.NativePdbRequiresDesktop)]
+        [WorkItem("https://github.com/dotnet/roslyn/issues/75237")]
+        [CombinatorialData]
+        public void NativeWriterLimit_Under([CombinatorialRange(SymUnmanagedWriterImpl.CustomMetadataByteLimit - 9, 10)] int length)
+        {
+            CompileWithMockedCustomMetadata(length).Diagnostics.Verify();
+        }
+
+        /// <summary>
+        /// Verifies the constant <c>SymUnmanagedWriterImpl.CustomMetadataByteLimit</c> against the external sym writer library we depend on.
+        /// </summary>
+        [ConditionalTheory(typeof(WindowsOnly), Reason = ConditionalSkipReason.NativePdbRequiresDesktop)]
+        [WorkItem("https://github.com/dotnet/roslyn/issues/75237")]
+        [CombinatorialData]
+        public void NativeWriterLimit_Over([CombinatorialRange(SymUnmanagedWriterImpl.CustomMetadataByteLimit + 1, 10)] int length)
+        {
+            CompileWithMockedCustomMetadata(length).Diagnostics.Verify(
+                // error CS0041: Unexpected error writing debug information -- 'Cannot emit native PDB for method 'C.M()' because its debug metadata size 65505 is over the limit 65504.'
+                Diagnostic(ErrorCode.FTL_DebugEmitFailure).WithArguments(string.Format(CodeAnalysisResources.SymWriterMetadataOverLimit, "C.M()", length, 65504)).WithLocation(1, 1));
+        }
+
+        private static EmitResult CompileWithMockedCustomMetadata(int length)
+        {
+            var comp = CreateCompilation("""
+                class C
+                {
+                    void M() { }
+                }
+                """);
+            return comp.Emit(
+                peStream: new MemoryStream(),
+                metadataPEStream: null,
+                pdbStream: new MemoryStream(),
+                xmlDocumentationStream: null,
+                cancellationToken: default,
+                win32Resources: null,
+                manifestResources: null,
+                options: null,
+                debugEntryPoint: null,
+                sourceLinkStream: null,
+                embeddedTexts: null,
+                rebuildData: null,
+                testData: new CompilationTestData
+                {
+                    SymWriterFactory = metadataProvider =>
+                    {
+                        var writer = SymWriterTestUtilities.CreateUnmanagedWriter(metadataProvider);
+                        return new CustomMetadataSymUnmanagedWriter(writer, new byte[length]);
+                    },
+                });
+        }
+
+        [ConditionalFact(typeof(WindowsOnly), Reason = ConditionalSkipReason.NativePdbRequiresDesktop)]
+        [WorkItem("https://github.com/dotnet/roslyn/issues/75237")]
+        public void NativeWriterLimit_EndToEnd()
+        {
+            var locals = Enumerable.Range(0, 14_000)
+                .Select(i => $"""
+                    var local{i} = {i};
+                    M2(local{i});
+                    """)
+                .Join(Environment.NewLine);
+            var source = $$"""
+                namespace N;
+                class C
+                {
+                    void M1()
+                    {
+                        {{locals}}
+                    }
+                    void M2(int x) { }
+                }
+                """;
+            CreateCompilation(source, options: TestOptions.DebugDll).VerifyEmitDiagnostics(
+                // error CS0041: Unexpected error writing debug information -- 'Cannot emit native PDB for method 'N.C.M1()' because its debug metadata size 69096 is over the limit 65504.'
+                Diagnostic(ErrorCode.FTL_DebugEmitFailure).WithArguments(string.Format(CodeAnalysisResources.SymWriterMetadataOverLimit, "N.C.M1()", 69096, 65504)).WithLocation(1, 1));
+        }
+
         [WorkItem(1067635, "http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/1067635")]
         [Fact]
         public void SuppressDynamicAndEncCDIForWinRT()

src/Compilers/Core/Portable/CodeAnalysisResources.resx

@@ -507,6 +507,9 @@
   <data name="SymWriterDoesNotSupportSourceLink" xml:space="preserve">
     <value>Windows PDB writer doesn't support SourceLink feature: '{0}'</value>
   </data>
+  <data name="SymWriterMetadataOverLimit" xml:space="preserve">
+    <value>Cannot emit native PDB for method '{0}' because its debug metadata size {1} is over the limit {2}.</value>
+  </data>
   <data name="RuleSetBadAttributeValue" xml:space="preserve">
     <value>The attribute {0} has an invalid value of {1}.</value>
   </data>

src/Compilers/Core/Portable/DiaSymReader/Writer/SymUnmanagedWriter.cs

@@ -7,6 +7,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using Microsoft.Cci;
 
 namespace Microsoft.DiaSymReader
 {
@@ -141,7 +142,7 @@ public abstract void SetAsyncInfo(
         /// <exception cref="InvalidOperationException">Writes are not allowed to the underlying stream.</exception>
         /// <exception cref="SymUnmanagedWriterException">Error occurred while writing PDB data.</exception>
         /// <exception cref="ArgumentNullException"><paramref name="metadata"/> is null</exception>
-        public abstract void DefineCustomMetadata(byte[] metadata);
+        public abstract void DefineCustomMetadata(byte[] metadata, IMethodDefinition methodDefinition);
 
         /// <summary>
         /// Designates specified method as an entry point.

src/Compilers/Core/Portable/DiaSymReader/Writer/SymUnmanagedWriterImpl.cs

@@ -4,6 +4,8 @@
 
 #nullable disable
 
+using Microsoft.Cci;
+using Microsoft.CodeAnalysis;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -16,6 +18,9 @@ namespace Microsoft.DiaSymReader
 {
     internal sealed class SymUnmanagedWriterImpl : SymUnmanagedWriter
     {
+        // This constant is verified in PDBTests.NativeWriterLimit_Under and NativeWriterLimit_Over.
+        internal const int CustomMetadataByteLimit = 65_504;
+
         private static readonly object s_zeroInt32 = 0;
 
         private ISymUnmanagedWriter5 _symWriter;
@@ -532,7 +537,7 @@ public override void SetAsyncInfo(
             }
         }
 
-        public override unsafe void DefineCustomMetadata(byte[] metadata)
+        public override unsafe void DefineCustomMetadata(byte[] metadata, IMethodDefinition methodDefinition)
         {
             if (metadata == null)
             {
@@ -544,6 +549,15 @@ public override unsafe void DefineCustomMetadata(byte[] metadata)
                 return;
             }
 
+            if (metadata.Length > CustomMetadataByteLimit)
+            {
+                throw new SymUnmanagedWriterException(string.Format(
+                    CodeAnalysisResources.SymWriterMetadataOverLimit,
+                    methodDefinition,
+                    metadata.Length,
+                    CustomMetadataByteLimit));
+            }
+
             var symWriter = GetSymWriter();
 
             try

src/Compilers/Core/Portable/NativePdbWriter/PdbWriter.cs

@@ -148,7 +148,7 @@ public void SerializeDebugInfo(IMethodBody methodBody, StandaloneSignatureHandle
 
             if (blob.Length > 0)
             {
-                _symWriter.DefineCustomMetadata(blob);
+                _symWriter.DefineCustomMetadata(blob, methodBody.MethodDefinition);
             }
 
             CloseMethod(methodBody.IL.Length);