Vector abi tests and reverse pinvoke fixes (#32034)

- Fix issues noted in returning structures from reverse p/invokes

* Add tests for marshalling Vector2/3/4
- Anticipated work on Windows Amd64 calling conventions may affect this, so this needs testing
* Updates to add return struct marshalling tests/fixes
- Adjust some reverse p/invoke struct marshalling tests to add coverage for returning structures
  - Coverage includes a 3 byte, 4 byte struct, 8 byte struct, and a large struct
  - Note: There is a code in mlinfo.cpp which explicitly makes support for returning 3, 5, 6, and 7 on X86 follow the wrong calling convention by rounding the size to a 4 byte boundary. I've left this unfortunate detail in place.
* Re-encode ReversePInvokeTest.cs to utf8 to reduce future diffing burden

src/coreclr/src/vm/dllimport.cpp

@@ -3983,10 +3983,10 @@ static void CreateNDirectStubWorker(StubState*         pss,
         // For functions with value type class, managed and unmanaged calling convention differ
         fMarshalReturnValueFirst = HasRetBuffArgUnmanagedFixup(&msig);
 #elif defined(TARGET_ARM)
-        fMarshalReturnValueFirst = HasRetBuffArg(&msig);
+        fMarshalReturnValueFirst = (isInstanceMethod && isReturnTypeValueType) && HasRetBuffArg(&msig);
 #else
         // On Windows-X86, the native signature might need a return buffer when the managed doesn't (specifically when the native signature is a member function).
-        fMarshalReturnValueFirst = HasRetBuffArg(&msig) || (isInstanceMethod && isReturnTypeValueType);
+        fMarshalReturnValueFirst = (!SF_IsReverseStub(dwStubFlags) && HasRetBuffArg(&msig)) || (isInstanceMethod && isReturnTypeValueType);
 #endif // UNIX_X86_ABI
 #elif defined(TARGET_AMD64) || defined (TARGET_ARM64)
         fMarshalReturnValueFirst = isInstanceMethod && isReturnTypeValueType;

src/coreclr/tests/src/Interop/CMakeLists.txt

@@ -34,6 +34,7 @@ add_subdirectory(PInvoke/CriticalHandles)
 add_subdirectory(PInvoke/Generics)
 add_subdirectory(PInvoke/AsAny)
 add_subdirectory(PInvoke/SafeHandles)
+add_subdirectory(PInvoke/Vector2_3_4)
 add_subdirectory(NativeCallable)
 add_subdirectory(PrimitiveMarshalling/Bool)
 add_subdirectory(PrimitiveMarshalling/UIntPtr)

src/coreclr/tests/src/Interop/PInvoke/Vector2_3_4/CMakeLists.txt

@@ -0,0 +1,8 @@
+
+include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") 
+set(SOURCES 
+    Vector2_3_4TestNative.cpp 
+)
+add_library (Vector2_3_4TestNative SHARED ${SOURCES}) 
+target_link_libraries(Vector2_3_4TestNative ${LINK_LIBRARIES_ADDITIONAL})
+install (TARGETS Vector2_3_4TestNative DESTINATION bin) 

src/coreclr/tests/src/Interop/PInvoke/Vector2_3_4/Vector2_3_4.cs

@@ -0,0 +1,153 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Numerics;
+using System.Runtime.InteropServices;
+using TestLibrary;
+
+public class Vector2_3_4Test
+{
+    private const int StartingIntValue = 42;
+    private const int NewIntValue = 18;
+
+    public static int Main()
+    {
+        try
+        {
+            RunVector2Tests();
+            RunVector3Tests();
+            RunVector4Tests();
+        }
+        catch (System.Exception ex)
+        {
+            Console.WriteLine(ex.ToString());
+            return 101;
+        }
+        return 100;
+    }
+
+    private static void RunVector2Tests()
+    {
+        float X = StartingIntValue;
+        float Y = StartingIntValue + 1;
+        float Z = StartingIntValue + 232;
+        float W = StartingIntValue + 93719;
+
+        float XNew = 71;
+        float YNew = 999;
+        float ZNew = 1203;
+        float WNew = 4;
+
+        Vector2 startingVector = new Vector2(X, Y);
+        Vector2 newVector = new Vector2(XNew, YNew);
+
+        Assert.AreEqual(startingVector, Vector2_3_4TestNative.CreateVector2FromFloats(X, Y));
+
+        Assert.IsTrue(Vector2_3_4TestNative.Vector2EqualToFloats(startingVector, X, Y));
+
+        Vector2 localVector = startingVector;
+        Assert.IsTrue(Vector2_3_4TestNative.ValidateAndChangeVector2(ref localVector, X, Y, XNew, YNew));
+        Assert.AreEqual(newVector, localVector);
+
+        Vector2_3_4TestNative.GetVector2ForFloats(X, Y, out var vec);
+        Assert.AreEqual(startingVector, vec);
+
+        Assert.AreEqual(startingVector, Vector2_3_4TestNative.CreateWrappedVector2FromFloats(X, Y).vec);
+
+        Assert.IsTrue(Vector2_3_4TestNative.WrappedVector2EqualToFloats(new Vector2_3_4TestNative.Vector2Wrapper { vec = startingVector }, X, Y));
+
+        var localVectorWrapper = new Vector2_3_4TestNative.Vector2Wrapper { vec = startingVector };
+        Assert.IsTrue(Vector2_3_4TestNative.ValidateAndChangeWrappedVector2(ref localVectorWrapper, X, Y, XNew, YNew));
+        Assert.AreEqual(newVector, localVectorWrapper.vec);
+
+        Assert.AreEqual(newVector, Vector2_3_4TestNative.PassThroughVector2ToCallback(startingVector, vectorParam => 
+        {
+            Assert.AreEqual(startingVector, vectorParam);
+            return newVector;
+        }));
+    }
+
+    private static void RunVector3Tests()
+    {
+        float X = StartingIntValue;
+        float Y = StartingIntValue + 1;
+        float Z = StartingIntValue + 232;
+        float W = StartingIntValue + 93719;
+
+        float XNew = 71;
+        float YNew = 999;
+        float ZNew = 1203;
+        float WNew = 4;
+
+        Vector3 startingVector = new Vector3(X, Y, Z);
+        Vector3 newVector = new Vector3(XNew, YNew, ZNew);
+
+        Assert.AreEqual(startingVector, Vector2_3_4TestNative.CreateVector3FromFloats(X, Y, Z));
+
+        Assert.IsTrue(Vector2_3_4TestNative.Vector3EqualToFloats(startingVector, X, Y, Z));
+
+        Vector3 localVector = startingVector;
+        Assert.IsTrue(Vector2_3_4TestNative.ValidateAndChangeVector3(ref localVector, X, Y, Z, XNew, YNew, ZNew));
+        Assert.AreEqual(newVector, localVector);
+
+        Vector2_3_4TestNative.GetVector3ForFloats(X, Y, Z, out var vec);
+        Assert.AreEqual(startingVector, vec);
+
+        Assert.AreEqual(startingVector, Vector2_3_4TestNative.CreateWrappedVector3FromFloats(X, Y, Z).vec);
+
+        Assert.IsTrue(Vector2_3_4TestNative.WrappedVector3EqualToFloats(new Vector2_3_4TestNative.Vector3Wrapper { vec = startingVector }, X, Y, Z));
+
+        var localVectorWrapper = new Vector2_3_4TestNative.Vector3Wrapper { vec = startingVector };
+        Assert.IsTrue(Vector2_3_4TestNative.ValidateAndChangeWrappedVector3(ref localVectorWrapper, X, Y, Z, XNew, YNew, ZNew));
+        Assert.AreEqual(newVector, localVectorWrapper.vec);
+
+        Assert.AreEqual(newVector, Vector2_3_4TestNative.PassThroughVector3ToCallback(startingVector, vectorParam => 
+        {
+            Assert.AreEqual(startingVector, vectorParam);
+            return newVector;
+        }));
+    }
+
+    private static void RunVector4Tests()
+    {
+        float X = StartingIntValue;
+        float Y = StartingIntValue + 1;
+        float Z = StartingIntValue + 232;
+        float W = StartingIntValue + 93719;
+
+        float XNew = 71;
+        float YNew = 999;
+        float ZNew = 1203;
+        float WNew = 4;
+
+        Vector4 startingVector = new Vector4(X, Y, Z, W);
+        Vector4 newVector = new Vector4(XNew, YNew, ZNew, WNew);
+
+        Assert.AreEqual(startingVector, Vector2_3_4TestNative.CreateVector4FromFloats(X, Y, Z, W));
+
+        Assert.IsTrue(Vector2_3_4TestNative.Vector4EqualToFloats(startingVector, X, Y, Z, W));
+
+        Vector4 localVector = startingVector;
+        Assert.IsTrue(Vector2_3_4TestNative.ValidateAndChangeVector4(ref localVector, X, Y, Z, W, XNew, YNew, ZNew, WNew));
+        Assert.AreEqual(newVector, localVector);
+
+        Vector2_3_4TestNative.GetVector4ForFloats(X, Y, Z, W, out var vec);
+        Assert.AreEqual(startingVector, vec);
+
+        Assert.AreEqual(startingVector, Vector2_3_4TestNative.CreateWrappedVector4FromFloats(X, Y, Z, W).vec);
+
+        Assert.IsTrue(Vector2_3_4TestNative.WrappedVector4EqualToFloats(new Vector2_3_4TestNative.Vector4Wrapper { vec = startingVector }, X, Y, Z, W));
+
+        var localVectorWrapper = new Vector2_3_4TestNative.Vector4Wrapper { vec = startingVector };
+        Assert.IsTrue(Vector2_3_4TestNative.ValidateAndChangeWrappedVector4(ref localVectorWrapper, X, Y, Z, W, XNew, YNew, ZNew, WNew));
+        Assert.AreEqual(newVector, localVectorWrapper.vec);
+
+        Assert.AreEqual(newVector, Vector2_3_4TestNative.PassThroughVector4ToCallback(startingVector, vectorParam => 
+        {
+            Assert.AreEqual(startingVector, vectorParam);
+            return newVector;
+        }));
+    }
+}

src/coreclr/tests/src/Interop/PInvoke/Vector2_3_4/Vector2_3_4.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="$([MSBuild]::GetPathOfFileAbove(Interop.settings.targets))" />
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="*.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="CMakeLists.txt" />
+  </ItemGroup>
+</Project>

src/coreclr/tests/src/Interop/PInvoke/Vector2_3_4/Vector2_3_4TestNative.cpp

@@ -0,0 +1,182 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include "xplatform.h"
+#include <new>
+
+struct Vector2
+{
+    float x;
+    float y;
+};
+
+struct Vector3
+{
+    float x;
+    float y;
+    float z;
+};
+
+struct Vector4
+{
+    float x;
+    float y;
+    float z;
+    float w;
+};
+
+namespace
+{
+    BOOL operator==(Vector2 lhs, Vector2 rhs)
+    {
+        return lhs.x == rhs.x && lhs.y == rhs.y ? TRUE : FALSE;
+    }
+    BOOL operator==(Vector3 lhs, Vector3 rhs)
+    {
+        return lhs.x == rhs.x && lhs.y == rhs.y && lhs.z == rhs.z ? TRUE : FALSE;
+    }
+    BOOL operator==(Vector4 lhs, Vector4 rhs)
+    {
+        return lhs.x == rhs.x && lhs.y == rhs.y && lhs.z == rhs.z && lhs.w == rhs.w ? TRUE : FALSE;
+    }
+}
+
+extern "C" DLL_EXPORT Vector4 STDMETHODCALLTYPE CreateVector4FromFloats(float x, float y, float z, float w)
+{
+    Vector4 result;
+    result.x = x;
+    result.y = y;
+    result.z = z;
+    result.w = w;
+    return result;
+}
+
+extern "C" DLL_EXPORT Vector3 STDMETHODCALLTYPE CreateVector3FromFloats(float x, float y, float z)
+{
+    Vector3 result;
+    result.x = x;
+    result.y = y;
+    result.z = z;
+    return result;
+}
+
+extern "C" DLL_EXPORT Vector2 STDMETHODCALLTYPE CreateVector2FromFloats(float x, float y)
+{
+    Vector2 result;
+    result.x = x;
+    result.y = y;
+    return result;
+}
+
+extern "C" DLL_EXPORT BOOL STDMETHODCALLTYPE Vector4EqualToFloats(Vector4 vec, float x, float y, float z, float w)
+{
+    Vector4 vecFromFloats;
+    vecFromFloats.x = x;
+    vecFromFloats.y = y;
+    vecFromFloats.z = z;
+    vecFromFloats.w = w;
+    return vec == vecFromFloats;
+}
+
+extern "C" DLL_EXPORT BOOL STDMETHODCALLTYPE Vector3EqualToFloats(Vector3 vec, float x, float y, float z)
+{
+    Vector3 vecFromFloats;
+    vecFromFloats.x = x;
+    vecFromFloats.y = y;
+    vecFromFloats.z = z;
+    return vec == vecFromFloats;
+}
+
+extern "C" DLL_EXPORT BOOL STDMETHODCALLTYPE Vector2EqualToFloats(Vector2 vec, float x, float y)
+{
+    Vector2 vecFromFloats;
+    vecFromFloats.x = x;
+    vecFromFloats.y = y;
+    return vec == vecFromFloats;
+}
+
+extern "C" DLL_EXPORT BOOL STDMETHODCALLTYPE ValidateAndChangeVector4(Vector4* vec, float expectedX, float expectedY, float expectedZ, float expectedW, float newX, float newY, float newZ, float newW)
+{
+    Vector4 vecExpected;
+    vecExpected.x = expectedX;
+    vecExpected.y = expectedY;
+    vecExpected.z = expectedZ;
+    vecExpected.w = expectedW;
+
+    BOOL result = *vec == vecExpected;
+    vec->x = newX;
+    vec->y = newY;
+    vec->z = newZ;
+    vec->w = newW;
+    return result;
+}
+
+extern "C" DLL_EXPORT BOOL STDMETHODCALLTYPE ValidateAndChangeVector3(Vector3* vec, float expectedX, float expectedY, float expectedZ, float newX, float newY, float newZ)
+{
+    Vector3 vecExpected;
+    vecExpected.x = expectedX;
+    vecExpected.y = expectedY;
+    vecExpected.z = expectedZ;
+
+    BOOL result = *vec == vecExpected;
+    vec->x = newX;
+    vec->y = newY;
+    vec->z = newZ;
+    return result;
+}
+
+extern "C" DLL_EXPORT BOOL STDMETHODCALLTYPE ValidateAndChangeVector2(Vector2* vec, float expectedX, float expectedY, float newX, float newY)
+{
+    Vector2 vecExpected;
+    vecExpected.x = expectedX;
+    vecExpected.y = expectedY;
+
+    BOOL result = *vec == vecExpected;
+    vec->x = newX;
+    vec->y = newY;
+    return result;
+}
+
+extern "C" DLL_EXPORT void STDMETHODCALLTYPE GetVector4ForFloats(float x, float y, float z, float w, Vector4* vec)
+{
+    vec->x = x;
+    vec->y = y;
+    vec->z = z;
+    vec->w = w;
+}
+
+extern "C" DLL_EXPORT void STDMETHODCALLTYPE GetVector3ForFloats(float x, float y, float z, Vector3* vec)
+{
+    vec->x = x;
+    vec->y = y;
+    vec->z = z;
+}
+
+extern "C" DLL_EXPORT void STDMETHODCALLTYPE GetVector2ForFloats(float x, float y, Vector2* vec)
+{
+    vec->x = x;
+    vec->y = y;
+}
+
+
+using Vector4Callback = Vector4(STDMETHODCALLTYPE*)(Vector4);
+
+extern "C" DLL_EXPORT Vector4 STDMETHODCALLTYPE PassThroughVector4ToCallback(Vector4 vec, Vector4Callback cb)
+{
+    return cb(vec);
+}
+
+using Vector3Callback = Vector3(STDMETHODCALLTYPE*)(Vector3);
+
+extern "C" DLL_EXPORT Vector3 STDMETHODCALLTYPE PassThroughVector3ToCallback(Vector3 vec, Vector3Callback cb)
+{
+    return cb(vec);
+}
+
+using Vector2Callback = Vector2(STDMETHODCALLTYPE*)(Vector2);
+
+extern "C" DLL_EXPORT Vector2 STDMETHODCALLTYPE PassThroughVector2ToCallback(Vector2 vec, Vector2Callback cb)
+{
+    return cb(vec);
+}