Convert CustomMarshalerInfo GetInstance to UnmanagedCallersOnly

src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -935,6 +936,40 @@ internal static void ClearManaged(ICustomMarshaler marshaler, in object pManaged
 
             marshaler.CleanUpManagedData(pManagedHome);
         }
+
+        [UnmanagedCallersOnly]
+        [UnconditionalSuppressMessage("Trimming", "IL2075", Justification = "Custom marshaler GetInstance method is preserved by ILLink (see MarkCustomMarshalerGetInstance).")]
+        internal static void GetCustomMarshalerInstance(void* pMT, byte* pCookie, int cCookieBytes, object* pResult, Exception* pException)
+        {
+            try
+            {
+                RuntimeType marshalerType = RuntimeTypeHandle.GetRuntimeType((MethodTable*)pMT);
+
+                MethodInfo? method = marshalerType.GetMethod(
+                    "GetInstance",
+                    Reflection.BindingFlags.Static | Reflection.BindingFlags.Public | Reflection.BindingFlags.NonPublic,
+                    [typeof(string)]);
+
+                if (method is null || typeof(ICustomMarshaler) != method.ReturnType)
+                {
+                    throw new ApplicationException(SR.Format(SR.CustomMarshaler_NoGetInstanceMethod, marshalerType.FullName));
+                }
+
+                var getInstance = method.CreateDelegate<Func<string, ICustomMarshaler>>();
+                string cookie = Text.Encoding.UTF8.GetString(new ReadOnlySpan<byte>(pCookie, cCookieBytes));
+                object? result = getInstance(cookie);
+                if (result is null)
+                {
+                    throw new ApplicationException(SR.Format(SR.CustomMarshaler_NullReturnForGetInstance, marshalerType.FullName));
+                }
+
+                *pResult = result;
+            }
+            catch (Exception ex)
+            {
+                *pException = ex;
+            }
+        }
     }  // class MngdRefCustomMarshaler
 
     internal struct AsAnyMarshaler

src/coreclr/dlls/mscorrc/mscorrc.rc

@@ -401,7 +401,6 @@ BEGIN
     IDS_EE_BAD_COMEXTENDS_CLASS             "Types extending from COM objects should override all methods of an interface implemented by the base COM class."
     IDS_EE_MARSHAL_UNMAPPABLE_CHAR          "Cannot marshal: Encountered unmappable character."
 
-    IDS_EE_NOCUSTOMMARSHALER                "A call to GetInstance() for custom marshaler '%1' returned null, which is not allowed."
     IDS_EE_SIZECONTROLOUTOFRANGE            "Array size control parameter index is out of range."
     IDS_EE_SIZECONTROLBADTYPE               "Array size control parameter type not supported."
     IDS_EE_SAFEARRAYSZARRAYMISMATCH         "SafeArray cannot be marshaled to this array type because it has either nonzero lower bounds or more than one dimension."
@@ -462,7 +461,6 @@ BEGIN
     IDS_EE_BAD_COMEVENTITF_CLASS            "Methods on COM event interfaces must take a single delegate derived class as a parameter and have a void return type."
 
     IDS_EE_ICUSTOMMARSHALERNOTIMPL          "Custom marshaler '%1' does not implement the ICustomMarshaler interface."
-    IDS_EE_GETINSTANCENOTIMPL               "Custom marshaler '%1' does not implement a static GetInstance method that takes a single string parameter and returns an ICustomMarshaler."
 
     IDS_CLASSLOAD_COMIMPCANNOTHAVELAYOUT    "COM import type '%1' from assembly '%2' cannot have layout information."
     IDS_EE_INVALIDCOMDEFITF                 "Type '%1' has an invalid default COM interface: '%2'."

src/coreclr/dlls/mscorrc/resource.h

@@ -180,7 +180,6 @@
 #define IDS_EE_NDIRECT_BADNATL_CALLCONV         0x17df
 #define IDS_EE_CANNOTCAST                       0x17e0
 
-#define IDS_EE_NOCUSTOMMARSHALER                0x17e7
 #define IDS_EE_SIZECONTROLOUTOFRANGE            0x17e8
 #define IDS_EE_SIZECONTROLBADTYPE               0x17e9
 #define IDS_EE_SAFEARRAYSZARRAYMISMATCH         0x17eb
@@ -217,7 +216,6 @@
 #define IDS_EE_BAD_COMEVENTITF_CLASS            0x1a25
 
 #define IDS_EE_ICUSTOMMARSHALERNOTIMPL          0x1a2e
-#define IDS_EE_GETINSTANCENOTIMPL               0x1a2f
 
 #define IDS_EE_BADMARSHAL_CUSTOMMARSHALER       0x1a30
 

src/coreclr/vm/corelib.h

@@ -1171,6 +1171,7 @@ DEFINE_METHOD(MNGD_REF_CUSTOM_MARSHALER, CLEAR_MANAGED,               ClearManag
 DEFINE_METHOD(MNGD_REF_CUSTOM_MARSHALER, CONVERT_CONTENTS_TO_NATIVE_UCO,  ConvertContentsToNative,    SM_PtrICustomMarshaler_PtrObj_PtrIntPtr_PtrException_RetVoid)
 DEFINE_METHOD(MNGD_REF_CUSTOM_MARSHALER, CONVERT_CONTENTS_TO_MANAGED_UCO, ConvertContentsToManaged,   SM_PtrICustomMarshaler_PtrObj_PtrIntPtr_PtrException_RetVoid)
 DEFINE_METHOD(MNGD_REF_CUSTOM_MARSHALER, CLEAR_MANAGED_UCO,               ClearManaged,               SM_PtrICustomMarshaler_PtrObj_PtrIntPtr_PtrException_RetVoid)
+DEFINE_METHOD(MNGD_REF_CUSTOM_MARSHALER, GET_CUSTOM_MARSHALER_INSTANCE,    GetCustomMarshalerInstance,  SM_PtrVoid_PtrByte_Int_PtrObj_PtrException_RetVoid)
 
 DEFINE_CLASS(ASANY_MARSHALER,            StubHelpers,                 AsAnyMarshaler)
 DEFINE_METHOD(ASANY_MARSHALER,           CTOR,                        .ctor,                      IM_IntPtr_RetVoid)

src/coreclr/vm/custommarshalerinfo.cpp

@@ -16,53 +16,7 @@
 #include "custommarshalerinfo.h"
 #include "mlinfo.h"
 #include "sigbuilder.h"
-
-namespace
-{
-    MethodDesc * FindGetInstanceMethod(TypeHandle hndCustomMarshalerType)
-    {
-        CONTRACTL
-        {
-            THROWS;
-            GC_TRIGGERS;
-            MODE_COOPERATIVE;
-        }
-        CONTRACTL_END;
-
-
-        MethodTable *pMT = hndCustomMarshalerType.AsMethodTable();
-
-        MethodDesc *pMD = MemberLoader::FindMethod(pMT, "GetInstance", &gsig_SM_Str_RetICustomMarshaler);
-        if (!pMD)
-        {
-            DefineFullyQualifiedNameForClassW()
-            COMPlusThrow(kApplicationException,
-                        IDS_EE_GETINSTANCENOTIMPL,
-                        GetFullyQualifiedNameForClassW(pMT));
-        };
-
-        // If the GetInstance method is generic, get an instantiating stub for it -
-        // the CallDescr infrastructure doesn't know how to pass secret generic arguments.
-        if (pMD->RequiresInstMethodTableArg())
-        {
-            pMD = MethodDesc::FindOrCreateAssociatedMethodDesc(
-                pMD,
-                pMT,
-                FALSE,           // forceBoxedEntryPoint
-                Instantiation(), // methodInst
-                FALSE,           // allowInstParam
-                FALSE);          // forceRemotableMethod
-
-            _ASSERTE(!pMD->RequiresInstMethodTableArg());
-        }
-
-        // Ensure that the value types in the signature are loaded.
-        MetaSig::EnsureSigValueTypesLoaded(pMD);
-
-        // Return the specified method desc.
-        return pMD;
-    }
-}
+#include "callhelpers.h"
 
 //==========================================================================
 // Implementation of the custom marshaler info class.
@@ -99,36 +53,16 @@ CustomMarshalerInfo::CustomMarshalerInfo(LoaderAllocator *pLoaderAllocator, Type
     hndCustomMarshalerType.GetMethodTable()->EnsureInstanceActive();
     hndCustomMarshalerType.GetMethodTable()->CheckRunClassInitThrowing();
 
-    // Create a .NET string that will contain the string cookie.
-    STRINGREF CookieStringObj = StringObject::NewString(strCookie, cCookieStrBytes);
-    GCPROTECT_BEGIN(CookieStringObj);
-    // Load the method desc for the static method to retrieve the instance.
-    MethodDesc *pGetCustomMarshalerMD = FindGetInstanceMethod(hndCustomMarshalerType);
-
-    MethodDescCallSite getCustomMarshaler(pGetCustomMarshalerMD, (OBJECTREF*)&CookieStringObj);
-
-    pGetCustomMarshalerMD->EnsureActive();
-
-    // Prepare the arguments that will be passed to GetCustomMarshaler.
-    ARG_SLOT GetCustomMarshalerArgs[] = {
-        ObjToArgSlot(CookieStringObj)
-    };
-
-    // Call the GetCustomMarshaler method to retrieve the custom marshaler to use.
+    // Call the GetInstance method to retrieve the custom marshaler to use.
     OBJECTREF CustomMarshalerObj = NULL;
     GCPROTECT_BEGIN(CustomMarshalerObj);
-    CustomMarshalerObj = getCustomMarshaler.Call_RetOBJECTREF(GetCustomMarshalerArgs);
-    if (!CustomMarshalerObj)
-    {
-        DefineFullyQualifiedNameForClassW()
-        COMPlusThrow(kApplicationException,
-                     IDS_EE_NOCUSTOMMARSHALER,
-                     GetFullyQualifiedNameForClassW(hndCustomMarshalerType.GetMethodTable()));
-    }
+
+    UnmanagedCallersOnlyCaller getCustomMarshaler(METHOD__MNGD_REF_CUSTOM_MARSHALER__GET_CUSTOM_MARSHALER_INSTANCE);
+    getCustomMarshaler.InvokeThrowing(hndCustomMarshalerType.AsMethodTable(), (BYTE*)strCookie, (INT32)cCookieStrBytes, &CustomMarshalerObj);
+    _ASSERTE(CustomMarshalerObj != NULL);
 
     m_hndCustomMarshaler = pLoaderAllocator->AllocateHandle(CustomMarshalerObj);
     GCPROTECT_END();
-    GCPROTECT_END();
 }
 
 

src/coreclr/vm/metasig.h

@@ -222,6 +222,7 @@ DEFINE_METASIG(SM(Obj_Int_RetVoid, j i,v))
 DEFINE_METASIG(SM(PtrVoid_Obj_RetObj, P(v) j, j))
 DEFINE_METASIG_T(SM(ICustomMarshaler_RefObj_PtrIntPtr_RetVoid, C(ICUSTOM_MARSHALER) r(j) P(I), v))
 DEFINE_METASIG_T(SM(PtrICustomMarshaler_PtrObj_PtrIntPtr_PtrException_RetVoid, P(C(ICUSTOM_MARSHALER)) P(j) P(I) P(C(EXCEPTION)), v))
+DEFINE_METASIG_T(SM(PtrVoid_PtrByte_Int_PtrObj_PtrException_RetVoid, P(v) P(b) i P(j) P(C(EXCEPTION)), v))
 
 DEFINE_METASIG(SM(Long_Long_RetLong, l l, l))
 DEFINE_METASIG(SM(ULong_ULong_RetULong, L L, L))
@@ -291,7 +292,6 @@ DEFINE_METASIG_T(IM(RuntimeTypeHandle_RefBool_RefIntPtr_RetVoid, g(RT_TYPE_HANDL
 
 
 DEFINE_METASIG(SM(Str_RetInt, s, i))
-DEFINE_METASIG_T(SM(Str_RetICustomMarshaler, s, C(ICUSTOM_MARSHALER)))
 DEFINE_METASIG(SM(Int_Str_RetIntPtr, i s, I))
 DEFINE_METASIG(SM(Int_Str_IntPtr_RetIntPtr, i s I, I))
 DEFINE_METASIG(SM(Int_Str_IntPtr_Int_RetVoid, i s I i, v))