Using python ctypes to call io_submit in Linux

Question

I'm trying to call io_submit using python ctypes. The code I'm writing is supposed to work on both 32 and 64-bit Intel/AMD architectures, but here I'll focus on 64 bits.

I have defined the following:

def PADDED64(type, name1, name2):
    return [(name1, type), (name2, type)]

def PADDEDptr64(type, name1, name2): 
    return [(name1, type)]

def PADDEDul64(name1, name2):
    return [(name1, ctypes.c_ulong)]

class IOVec(ctypes.Structure):
    _fields_ = [("iov_base", ctypes.c_void_p), ("iov_len", ctypes.c_size_t)]

class IOCBDataCommon64(ctypes.Structure):
    _fields_ = PADDEDptr64(ctypes.c_void_p, "buf", "__pad1") + \
        PADDEDul64("nbytes", "__pad2") + \
        [("offset", ctypes.c_longlong), ("__pad3", ctypes.c_longlong), ("flags", ctypes.c_uint), ("resfd", ctypes.c_uint)]

class IOCBDataVector(ctypes.Structure):
    _fields_ = [("vec", ctypes.POINTER(IOVec)), ("nr", ctypes.c_int), ("offset", ctypes.c_longlong)]

class IOCBDataPoll64(ctypes.Structure):
    _fields_ = PADDED64(ctypes.c_int, "events", "__pad1")

class SockAddr(ctypes.Structure):
    _fields_ = [("sa_family", ctypes.c_ushort), ("sa_data", ctypes.c_char * 14)]

class IOCBDataSockAddr(ctypes.Structure):
    _fields_ = [("addr", ctypes.POINTER(SockAddr)), ("len", ctypes.c_int)]

class IOCBDataUnion64(ctypes.Union):
    _fields_ = [("c", IOCBDataCommon64), ("v", IOCBDataVector), ("poll", IOCBDataPoll64), ("saddr", IOCBDataSockAddr)]

class IOCB64(ctypes.Structure):
    _fields_ = PADDEDptr64(ctypes.c_void_p, "data" , "__pad1") + \
        PADDED64(ctypes.c_uint, "key", "__pad2") + \
        [("aio_lio_opcode", ctypes.c_short), ("aio_reqprio", ctypes.c_short), ("aio_fildes", ctypes.c_int), ("u", IOCBDataUnion64)]

class Timespec(ctypes.Structure):
    _fields_ = [("tv_sec", ctypes.c_long), ("tv_nsec", ctypes.c_long)]

class IOEvent64(ctypes.Structure):
    _fields_ = PADDEDptr64(ctypes.c_void_p, "data", "__pad1") + \
        PADDEDptr64(ctypes.POINTER(IOCB64), "obj", "__pad2") + \
        PADDEDul64("res", "__pad3") + \
        PADDEDul64("res2", "__pad4")

I have a wrapper class called AIOCommands:

class AIOCommands:
    def __init__(self, aioCommandList):
        self.__commandList = aioCommandList
        self.__iocbs = (IOCB64 * len(self.__commandList))()
        for i in range(len(self.__commandList)):
            self.__commandList[i].initialize(self.__iocbs[i])
    def size(self):
        return len(self.__iocbs)
    def getIOCBArray(self):
        return self.__iocbs

I have defined the arguments and the return value of io_submit:

class Executor:
    def __init__(self, aioLibraryPath):
        self.__aio = ctypes.CDLL(aioLibraryPath)
        self.__aio.io_submit.argtypes = [self.aio_context_t, ctypes.c_long, ctypes.POINTER(ctypes.POINTER(IOCB64))]
        self.__aio.io_submit.restype = ctypes.c_long

Now, what should Executor.io_submit body look like? I tried:

def io_submit(self, aioContext, aioCommands):
    iocbPtr = ctypes.cast(aioCommands.getIOCBArray(), ctypes.POINTER(self.iocb_t))
    return self.__aio.io_submit(aioContext, aioCommands.size(), ctypes.byref(iocbPtr))

But I get a segmentation fault whenever the length of aioCommandList is greater than 1. When the list contains just 1 command, the code works as expected.

Could this be a problem with my structure definitions? I've tried to imitate the definitions in libaio.h (assuming only little-endian architectures will be supported):

#if defined(__i386__) /* little endian, 32 bits */
#define PADDED(x, y)    x; unsigned y
#define PADDEDptr(x, y) x; unsigned y
#define PADDEDul(x, y)  unsigned long x; unsigned y
#elif defined(__ia64__) || defined(__x86_64__) || defined(__alpha__)
#define PADDED(x, y)    x, y
#define PADDEDptr(x, y) x
#define PADDEDul(x, y)  unsigned long x
#elif defined(__powerpc64__) /* big endian, 64 bits */
#define PADDED(x, y)    unsigned y; x
#define PADDEDptr(x,y)  x
#define PADDEDul(x, y)  unsigned long x
#elif defined(__PPC__)  /* big endian, 32 bits */
#define PADDED(x, y)    unsigned y; x
#define PADDEDptr(x, y) unsigned y; x
#define PADDEDul(x, y)  unsigned y; unsigned long x
#elif defined(__s390x__) /* big endian, 64 bits */
#define PADDED(x, y)    unsigned y; x
#define PADDEDptr(x,y)  x
#define PADDEDul(x, y)  unsigned long x
#elif defined(__s390__) /* big endian, 32 bits */
#define PADDED(x, y)    unsigned y; x
#define PADDEDptr(x, y) unsigned y; x
#define PADDEDul(x, y)  unsigned y; unsigned long x
#else
#error  endian?
#endif

struct io_iocb_poll {
    PADDED(int events, __pad1);
};  /* result code is the set of result flags or -'ve errno */

struct io_iocb_sockaddr {
    struct sockaddr *addr;
    int     len;
};  /* result code is the length of the sockaddr, or -'ve errno */

struct io_iocb_common {
    PADDEDptr(void  *buf, __pad1);
    PADDEDul(nbytes, __pad2);
    long long   offset;
    long long   __pad3;
    unsigned    flags;
    unsigned    resfd;
};  /* result code is the amount read or -'ve errno */

struct io_iocb_vector {
    const struct iovec  *vec;
    int         nr;
    long long       offset;
};  /* result code is the amount read or -'ve errno */

struct iocb {
    PADDEDptr(void *data, __pad1);  /* Return in the io completion event */
    PADDED(unsigned key, __pad2);   /* For use in identifying io requests */

    short       aio_lio_opcode; 
    short       aio_reqprio;
    int     aio_fildes;

    union {
        struct io_iocb_common       c;
        struct io_iocb_vector       v;
        struct io_iocb_poll     poll;
        struct io_iocb_sockaddr saddr;
    } u;
};

Any help would be appreciated, I've been stuck on this for several hours.

Answer 1

The way I understand it is that the iocbpp argument to io_submit() is an array of pointers to struct iocb.

This seems to be reinforced with the Linux-specific example here: http://voinici.ceata.org/~sana/blog/?p=248 and by the EINVAL error documentation here: http://linux.die.net/man/2/io_submit (array subscripting takes precedence over dereferencing)

What you have provided to io_submit() is a reference to an array of struct iocb. You will surely get a segfault as io_submit dereferences bogus memory addresses as it iterates through the iocbpp array. The first element (index 0) will work fine since there is no memory offset to access it.

edit Another example here: http://www.xmailserver.org/eventfd-aio-test.c