Nginx proxy module and socket descriptor passing

Question

I'm implementing a server passing socket descriptors to worker processes for handling. It send them via UNIX sockets using the sendmsg system call. Server itself can listen on a UNIX socket or an INET socket. I have a problem putting it behind nginx. When my server runs on a INET socket, everything is OK. But proxing doesn't work through a UNIX socket. nginx reports:

readv() failed (104: Connection reset by peer) while reading upstream

A simple Python script receives data without a problem:

import sys
import socket
sock = socket.socket(socket.AF_UNIX)
sock.connect(sys.argv[1])
while 1:
    data = sock.recv(1024)
    if not data: break
    print data

Here is a minimal example. Note that uncommenting two commented lines solves the problem.

#include <time.h>
#include <sys/socket.h>
#include <sys/un.h>

const char response[] =
    "HTTP/1.0 200 OK\r\n"
    "Content-Type: text/plain\r\n"
    "Content-Length: 4\r\n"
    "\r\n"
    "test";

int main(int argc, char** argv)
{
    struct msghdr msg;
    msg.msg_name = 0;
    msg.msg_namelen = 0;
    struct iovec iov;
    char c;
    iov.iov_base = &c;
    iov.iov_len = 1;
    msg.msg_iov = &iov;
    msg.msg_iovlen = 1;
    char control[CMSG_SPACE(sizeof(int))];
    msg.msg_control = control;
    msg.msg_controllen = sizeof(control);
    struct cmsghdr* cmsg_ptr = CMSG_FIRSTHDR(&msg);
    int pair[2];
    socketpair(AF_UNIX, SOCK_STREAM, 0, pair);
    if (fork()) {
        close(pair[1]);
        cmsg_ptr->cmsg_level = SOL_SOCKET;
        cmsg_ptr->cmsg_type = SCM_RIGHTS;
        cmsg_ptr->cmsg_len = CMSG_LEN(sizeof(int));
        int listen_fd = socket(AF_UNIX, SOCK_STREAM, 0);
        unlink(argv[1]);
        struct sockaddr_un address;
        address.sun_family = AF_UNIX;
        strncpy(address.sun_path, argv[1], sizeof(address.sun_path) - 1);
        bind(listen_fd, (struct sockaddr*)(&address), SUN_LEN(&address));
        chmod(argv[1], 0666);
        listen(listen_fd, SOMAXCONN);
        for (;;) {
            int conn_fd = accept(listen_fd, 0, 0);
            *(int*)(CMSG_DATA(cmsg_ptr)) = conn_fd;
            sendmsg(pair[0], &msg, 0);
/*             sleep(1); */
            close(conn_fd);
        }
    } else {
        close(pair[0]);
        for (;;) {
            recvmsg(pair[1], &msg, 0);
            int conn_fd = *(int*)(CMSG_DATA(cmsg_ptr));
            write(conn_fd, response, sizeof(response));
/*             shutdown(conn_fd, SHUT_WR); */
            close(conn_fd);
        }
    }
    return 0;
}

I use nginx/0.7.62 on linux. Should I dig into nginx or do I misunderstand descriptor passing?

Answer 1

The sleep() isn't necessary - the shutdown() is right, though. The fix is that your child process should not close conn_fd until it has seen end-of-file from nginx:

write(conn_fd, response, ...);

/* Tell client no more data is forthcoming */
shutdown(conn_fd, SHUT_WR); 

/* Read until client also closes */
while (read(conn_fd, buffer, sizeof buffer) > 0)
{
    /* ... */
}

/* Now we can close */
close(conn_fd);

The "Connection reset by peer" is the OS informing nginx that your application closed the socket before it saw everything nginx sent - "everything" here includes the logical end of stream, not just the data bytes.