Add support for marshalling struct with delegates (#4737)

src/Common/src/TypeSystem/IL/Stubs/StructMarshallingThunk.cs

@@ -119,38 +119,6 @@ namespace Internal.IL.Stubs
             }
         }
 
-        public IEnumerable<InlineArrayCandidate> GetInlineArrayCandidates()
-        {
-            int index = 0;
-            MarshalAsDescriptor[] marshalAsDescriptors = ((MetadataType)ManagedType).GetFieldMarshalAsDescriptors();
-            foreach (FieldDesc field in ManagedType.GetFields())
-            {
-                if (field.IsStatic)
-                {
-                    continue;
-                }
-
-                Marshaller marshaller = _marshallers[index];
-
-                if (marshaller.MarshallerKind == MarshallerKind.ByValAnsiString
-                    || marshaller.MarshallerKind == MarshallerKind.ByValUnicodeString)
-                {
-                    yield return MarshalHelpers.GetInlineArrayCandidate(marshaller.ManagedType.Context.GetWellKnownType(WellKnownType.Char), marshaller.ElementMarshallerKind, _interopStateManager, marshalAsDescriptors[index]);
-                }
-                else if (marshaller.MarshallerKind == MarshallerKind.ByValArray
-                    || marshaller.MarshallerKind == MarshallerKind.ByValAnsiCharArray)
-                {
-                    var arrayType = marshaller.ManagedType as ArrayType;
-
-                    Debug.Assert(arrayType != null);
-
-                    yield return MarshalHelpers.GetInlineArrayCandidate(arrayType.ElementType, marshaller.ElementMarshallerKind, _interopStateManager, marshalAsDescriptors[index]);
-                }
-
-                index++;
-            }
-        }
-
         private Marshaller[] InitializeMarshallers()
         {
             Debug.Assert(_interopStateManager != null);

src/ILCompiler.Compiler/src/Compiler/CompilerGeneratedInteropStubManager.cs

@@ -136,6 +136,36 @@ namespace ILCompiler
                     };
             }
         }
+        
+        private void AddDependenciesDueToPInvokeStructDelegateField(ref DependencyList dependencies, NodeFactory factory, TypeDesc typeDesc)
+        {
+            if (typeDesc is ByRefType)
+            {
+                typeDesc = typeDesc.GetParameterType();
+            }
+
+            MetadataType metadataType = typeDesc as MetadataType;
+            if (metadataType != null) 
+            {
+                foreach (FieldDesc field in metadataType.GetFields())
+                {
+                    if (field.IsStatic)
+                    {
+                        continue;
+                    }
+                    TypeDesc fieldType = field.FieldType;
+
+                    if (fieldType.IsDelegate)
+                    {
+                        AddDependenciesDueToPInvokeDelegate(ref dependencies, factory, fieldType);
+                    }
+                    else if (MarshalHelpers.IsStructMarshallingRequired(fieldType))
+                    {
+                        AddDependenciesDueToPInvokeStructDelegateField(ref dependencies, factory, fieldType);
+                    }
+                }
+            }
+        }
 
         public override void AddDependeciesDueToPInvoke(ref DependencyList dependencies, NodeFactory factory, MethodDesc method)
         {
@@ -146,9 +176,19 @@ namespace ILCompiler
                 MethodSignature methodSig = method.Signature;
                 AddDependenciesDueToPInvokeDelegate(ref dependencies, factory, methodSig.ReturnType);
 
+                // struct may contain delegate fields, hence we need to add dependencies for it
+                if (MarshalHelpers.IsStructMarshallingRequired(methodSig.ReturnType))
+                {
+                    AddDependenciesDueToPInvokeStructDelegateField(ref dependencies, factory, methodSig.ReturnType);
+                }
+
                 for (int i = 0; i < methodSig.Length; i++)
                 {
                     AddDependenciesDueToPInvokeDelegate(ref dependencies, factory, methodSig[i]);
+                    if (MarshalHelpers.IsStructMarshallingRequired(methodSig[i]))
+                    {
+                        AddDependenciesDueToPInvokeStructDelegateField(ref dependencies, factory, methodSig[i]);
+                    }
                 }
             }
 
@@ -226,13 +266,7 @@ namespace ILCompiler
                 dependencies.Add(factory.MethodEntrypoint(GetStructMarshallingNativeToManagedStub(type)), "Struct Marshalling stub");
                 dependencies.Add(factory.MethodEntrypoint(GetStructMarshallingCleanupStub(type)), "Struct Marshalling stub");
 
-                foreach (var inlineArrayCandidate in stub.GetInlineArrayCandidates())
-                {
-                    foreach (var method in inlineArrayCandidate.ElementType.GetMethods())
-                    {
-                        dependencies.Add(factory.MethodEntrypoint(method), "inline array marshalling stub");
-                    }
-                }
+                AddDependenciesDueToPInvokeStructDelegateField(ref dependencies, factory, type);
             }
         }
 

tests/src/Simple/PInvoke/PInvoke.cs

@@ -199,6 +199,25 @@ namespace PInvokeTests
         [DllImport("*", CallingConvention = CallingConvention.StdCall)]
         static extern bool InlineStringTest(ref InlineString ias);
 
+        internal delegate int Callback0();
+        internal delegate int Callback1();
+        internal delegate int Callback2();
+
+        [DllImport("*")]
+        internal static extern bool RegisterCallbacks(ref Callbacks callbacks);
+
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct Callbacks
+        {
+            public Callback0 callback0;
+            public Callback1 callback1;
+            public Callback2 callback2;
+        }
+
+        public static int callbackFunc0() { return 0; }
+        public static int callbackFunc1() { return 1; }
+        public static int callbackFunc2() { return 2; }
+
         public static int Main(string[] args)
         {
             TestBlittableType();
@@ -621,7 +640,7 @@ namespace PInvokeTests
             ius.inlineString = "Hello World";
 
 
-            TestStruct2 ts = new TestStruct2() { f1 = 100, f2 = true };
+            TestStruct2 ts = new TestStruct2() { f1 = 100, f2 = true};
             int size = Marshal.SizeOf<TestStruct2>(ts);
             IntPtr memory = Marshal.AllocHGlobal(size);
             try
@@ -666,6 +685,12 @@ namespace PInvokeTests
                 pass = true;
             }
             ThrowIfNotEquals(true, pass, "Struct marshalling scenario6 failed.");
+
+            Callbacks callbacks = new Callbacks();
+            callbacks.callback0 = new Callback0(callbackFunc0);
+            callbacks.callback1 = new Callback1(callbackFunc1);
+            callbacks.callback2 = new Callback2(callbackFunc2);
+            ThrowIfNotEquals(true,  RegisterCallbacks(ref callbacks), "Scenario 7: Struct with delegate marshalling failed");
 #endif
         }
     }

tests/src/Simple/PInvoke/PInvokeNative.cpp

@@ -639,7 +639,18 @@ DLL_EXPORT  bool __stdcall InlineStringTest(inlineString* p)
     CopyAnsiString(p->inlineString, "Hello World!");
     return true;
 }
+struct Callbacks
+{
+    int(__stdcall *callback0) (void);
+    int(__stdcall *callback1) (void);
+    int(__stdcall *callback2) (void);
+};
+
+DLL_EXPORT bool __stdcall RegisterCallbacks(Callbacks *callbacks)
+{
+    return callbacks->callback0() == 0 && callbacks->callback1() == 1 && callbacks->callback2() == 2;
+}
 
 #if (_MSC_VER >= 1400)         // Check MSC version
 #pragma warning(pop)           // Renable previous depreciations
-#endif
+#endif