P/Invoke Marshalling Help - Nested Structures - VB.NET

Question

Ok, what the heck am I doing wrong here..

"System.TypeLoadException: Cannot marshal field 's2' of type 'MY_Struct1': The type definition of this field has layout information but has an invalid managed/unmanaged type combination or is unmarshalable."

'VENDORAPI short FunctionEx(Struct1* pstruct1);
 Declare Auto Function FunctionEx Lib "VENDORDLL.dll" (<MarshalAs(UnmanagedType.Struct)> ByRef pstruct1 As MY_Struct1) As Int16

Enum MY_ANEnum
    a
    b
    c
End Enum

Enum MY_ANEnum2
    x
    y
    z
End Enum

Const foo_size = 5
Const foo1_size = 15
Const foo2_size = 25
Const foo3_size = 35
Const superfoo_size = 10
Const superfoo_len = 20

<StructLayout(LayoutKind.Sequential)> _
Structure MY_Struct1
'typedef struct
'{
'short       Id;
'ANEnum      aEnum;
'Struct2     s2;
'} Struct1;
    <MarshalAs(UnmanagedType.I2)> Public Id As Int16
    <MarshalAs(UnmanagedType.I2)> Public aEnum As MY_ANEnum
    <MarshalAs(UnmanagedType.Struct)> Public s2 As MY_Struct2
End Structure

<StructLayout(LayoutKind.Sequential)> _
Structure MY_Struct2

'struct Struct2
'{
'Struct3   s3;
'Struct4   s4;
'Struct5   s5;
'Struct6   s6;
'};
    Public s3 As MY_Struct3
    Public s4 As MY_Struct4
    Public s5 As MY_Struct5
    Public s6 As MY_Struct6
End Structure

<StructLayout(LayoutKind.Sequential)> _
Structure MY_Struct3
'struct Struct3
'{
' short  Flag;
' long   Id;
' ANEnum2   aEnum2; 
' char   foo[foo_size];
'};
    <MarshalAs(UnmanagedType.I2)> Public Flag As Int16
    <MarshalAs(UnmanagedType.I4)> Public Id As Int32
    <MarshalAs(UnmanagedType.I2)> Public enum2 As MY_ANEnum2
    <MarshalAs(UnmanagedType.ByValTStr, SizeConst:=foo_size)> Public foo As String
End Structure

<StructLayout(LayoutKind.Sequential)> _
Structure MY_Struct4
'struct Struct4
'{
'char    foo1[foo1_size];
'char    foo2[foo2_size];
'};
    <MarshalAs(UnmanagedType.ByValTStr, SizeConst:=foo1_size)> Public foo1 As String
    <MarshalAs(UnmanagedType.ByValTStr, SizeConst:=foo2_size)> Public foo2 As String
End Structure

<StructLayout(LayoutKind.Sequential)> _
Structure MY_Struct5
'typedef struct
'{
'char    szfoo31[foo3_size];
'char    szfoo32[foo3_size];
'char    szfoo33[foo3_size];
'char    szfoo34[foo3_size];
'} Struct5;
    <MarshalAs(UnmanagedType.ByValTStr, SizeConst:=foo3_size)> Public szfoo31 As String
    <MarshalAs(UnmanagedType.ByValTStr, SizeConst:=foo3_size)> Public szfoo32 As String
    <MarshalAs(UnmanagedType.ByValTStr, SizeConst:=foo3_size)> Public szfoo33 As String
    <MarshalAs(UnmanagedType.ByValTStr, SizeConst:=foo3_size)> Public szfoo34 As String
End Structure

<StructLayout(LayoutKind.Sequential)> _
Structure MY_Struct6
'struct Struct6
'{
'short Id;
'char     aaszSuperfoo[superfoo_len][superfoo_size];
'short siId[superfoo_len];
'short siId2[superfoo_len];  
'};
    <MarshalAs(UnmanagedType.I2)> Public Id As Int16
    <MarshalAs(UnmanagedType.ByValArray, ArraySubType:=UnmanagedType.ByValTStr, SizeConst:=superfoo_len)> Public aaszSuperFoo() As String
    <MarshalAs(UnmanagedType.ByValArray, ArraySubType:=UnmanagedType.I2, SizeConst:=superfoo_len)> Public siId() As Int16
    <MarshalAs(UnmanagedType.ByValArray, ArraySubType:=UnmanagedType.I2, SizeConst:=superfoo_len)> Public siId2() As Int16
End Structure

I am making a guess here that it has something to do with the multi-dimensional array in MY_Struct6.

I tried just a simple build up and Marshal.SizeOf() to do a Marshal.StructureToPtr() directly as a test and it gives me a Size Cannot be calculated error.

All you P/Invoke ninjas, hook me up with an answer here, and not the "Just code it in C++" one please..

Thanks in advance.

Answer 1

Try the following definition.

Partial Public Class NativeConstants

    '''superfoo_size -> 10
    Public Const superfoo_size As Integer = 10

    '''superfoo_len -> 20
    Public Const superfoo_len As Integer = 20
End Class

<System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential, CharSet:=System.Runtime.InteropServices.CharSet.[Ansi])>  _
Public Structure Struct6

    '''short
    Public Id As Short

    '''char[200]
    <System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.ByValTStr, SizeConst:=200)>  _
    Public aaszSuperfoo As String

    '''short[20]
    <System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.ByValArray, SizeConst:=20, ArraySubType:=System.Runtime.InteropServices.UnmanagedType.I2)>  _
    Public siId() As Short

    '''short[20]
    <System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.ByValArray, SizeConst:=20, ArraySubType:=System.Runtime.InteropServices.UnmanagedType.I2)>  _
    Public siId2() As Short
End Structure