Add support for abstract SafeHandle types for by-value marshalling. (d…

…otnet/runtimelab#1066)

* Add support for abstract SafeHandle types for by-value marshalling.

* Add test for byref abstract SafeHandle.

* Add abstract by-ref test. Add details for failure case.

Commit migrated from dotnet/runtimelab@a406e64

src/libraries/System.Runtime.InteropServices/gen/DllImportGenerator/Marshalling/MarshallingGenerator.cs

@@ -237,6 +237,13 @@ private static IMarshallingGenerator CreateCore(
                     {
                         throw new MarshallingNotSupportedException(info, context);
                     }
+                    if (info.IsByRef && info.ManagedType.IsAbstract)
+                    {
+                        throw new MarshallingNotSupportedException(info, context)
+                        {
+                            NotSupportedDetails = Resources.SafeHandleByRefMustBeConcrete
+                        };
+                    }
                     return new SafeHandleMarshaller(options);
 
                 // Marshalling in new model.

src/libraries/System.Runtime.InteropServices/gen/DllImportGenerator/Resources.resx

@@ -259,6 +259,9 @@
   <data name="RefValuePropertyUnsupportedMessage" xml:space="preserve">
     <value>The 'Value' property on the native type '{0}' must not be a 'ref' or 'readonly ref' property.</value>
   </data>
+  <data name="SafeHandleByRefMustBeConcrete" xml:space="preserve">
+    <value>An abstract type derived from 'SafeHandle' cannot be marshalled by reference. The provided type must be concrete.</value>
+  </data>
   <data name="StackallocConstructorMustHaveStackBufferSizeConstantDescription" xml:space="preserve">
     <value>When constructor taking a Span&lt;byte&gt; is specified on the native type, the type must also have a public integer constant named StackBufferSize to provide the size of the stack-allocated buffer.</value>
   </data>

src/libraries/System.Runtime.InteropServices/gen/DllImportGenerator/TypePositionInfo.cs

@@ -312,11 +312,10 @@ static bool TryCreateTypeBasedMarshallingInfo(ITypeSymbol type, DefaultMarshalli
                 var conversion = compilation.ClassifyCommonConversion(type, compilation.GetTypeByMetadataName(TypeNames.System_Runtime_InteropServices_SafeHandle)!);
                 if (conversion.Exists 
                     && conversion.IsImplicit 
-                    && conversion.IsReference 
-                    && !type.IsAbstract)
+                    && (conversion.IsReference || conversion.IsIdentity))
                 {
                     bool hasAccessibleDefaultConstructor = false;
-                    if (type is INamedTypeSymbol named && named.InstanceConstructors.Length > 0)
+                    if (type is INamedTypeSymbol named && !named.IsAbstract && named.InstanceConstructors.Length > 0)
                     {
                         foreach (var ctor in named.InstanceConstructors)
                         {

src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/CodeSnippets.cs

@@ -843,6 +843,15 @@ public static partial void Method(
         {byRefKind} {typeName} p);
 }}";
 
+        public static string BasicParameterByValue(string typeName) => @$"
+using System.Runtime.InteropServices;
+partial class Test
+{{
+    [GeneratedDllImport(""DoesNotExist"")]
+    public static partial void Method(
+        {typeName} p);
+}}";
+
         public static string BasicReturnType(string typeName) => @$"
 using System.Runtime.InteropServices;
 partial class Test

src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/CompileFails.cs

@@ -92,6 +92,9 @@ public static IEnumerable<object[]> CodeSnippetsToCompile()
 
             // Custom type marshalling in arrays (complex case with Value property)
             yield return new object[] { CodeSnippets.ArrayMarshallingWithCustomStructElementWithValueProperty, 5, 0 };
+
+            // Abstract SafeHandle type by reference
+            yield return new object[] { CodeSnippets.BasicParameterWithByRefModifier("ref", "System.Runtime.InteropServices.SafeHandle"), 1, 0 };
         }
 
         [Theory]

src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/Compiles.cs

@@ -135,6 +135,7 @@ public static IEnumerable<object[]> CodeSnippetsToCompile()
 
             // SafeHandle
             yield return new[] { CodeSnippets.BasicParametersAndModifiers("Microsoft.Win32.SafeHandles.SafeFileHandle") };
+            yield return new[] { CodeSnippets.BasicParameterByValue("System.Runtime.InteropServices.SafeHandle") };
             yield return new[] { CodeSnippets.SafeHandleWithCustomDefaultConstructorAccessibility(privateCtor: false) };
             yield return new[] { CodeSnippets.SafeHandleWithCustomDefaultConstructorAccessibility(privateCtor: true) };