Why recv() in winsock client cannot get any data once httpRetransmition happens?

Question

Hi guys, I am trying to record the time between 'http request' package and 'http response' package.

I write an socket client using winsock. The code is below

        if (send(sock, request.c_str(), request.length(), 0) != request.length())
            die_with_error("send() sent a different number of bytes than expected");

        // Record the time of httpRequestSent
        ::QueryPerformanceCounter(&httpRequestSent);
        ::QueryPerformanceFrequency(&frequency);

        //get response
        response = "";
        resp_leng= BUFFERSIZE;
        http_leng= 381;
        while(resp_leng==BUFFERSIZE||http_leng>0)
        {
            resp_leng= recv(sock, (char*)&buffer, BUFFERSIZE, 0);
            http_leng= http_leng - resp_leng;
            if (resp_leng>0)
                response+= string(buffer).substr(0,resp_leng);
            //note: download lag is not handled in this code
        }

        ::QueryPerformanceCounter(&httpResponseGot);

        //display response
        cout << response << endl;

        // Display the HTTP duration
        httpDuration = (double)(httpResponseGot.QuadPart - httpRequestSent.QuadPart) / (double)frequency.QuadPart;
        printf("The HTTP duration is %lf\n", httpDuration);

The code works nicely except one situation: HTTP Retransmition. I used wireshark to monitor packages and found out once there is a retransmition the code seems block on recv(), but cannot get any data from the socket buffer. I wonder why would this happen. Could somebody explain the reasons? Any help will be appreciated.

Answer 1

You are not doing adequate error checking on the calls to send() and recv(). Try something like this instead:

char *req_ptr = request.c_str();
int req_leng = request.length();
int req_index = 0;

do
{
    int req_sent = send(sock, req_ptr, req_leng, 0);
    if (req_sent < 1)
    {
        if ((req_sent == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
            continue;
        die_with_error("send() failed"); 
    }

    req_ptr += req_sent;
    req_leng -= req_sent;
}
while (req_leng > 0);

// Record the time of httpRequestSent 
::QueryPerformanceCounter(&httpRequestSent); 
::QueryPerformanceFrequency(&frequency); 

//get response 
std::string response; 
int resp_leng = BUFFERSIZE; 

int http_leng = -1;
bool http_leng_needed = true;

do
{ 
    if (http_leng_needed)
    {
        std::string::size_type pos = response.find("\r\n\r\n");
        if (pos != std::string::npos)
        {
            std::string resp_hdrs = response.substr(0, pos);

            // look for a "Content-Length" header to see
            // if the server sends who many bytes are
            // being sent after the headers.  Note that
            // the server may use "Transfer-Encoding: chunked"
            // instead, which has no "Content-Length" header...
            //
            // I will leave this as an excercise for you to figure out...

            http_leng = ...;

            // in case body bytes have already been received...
            http_leng -= (response.length() - (pos+4));

            http_leng_needed = false;
        }
    }

    if (http_leng_needed)
      resp_leng = BUFFERSIZE;
    else
      resp_leng = min(http_leng, BUFFERSIZE);

    if (resp_leng == 0)
        break;

    resp_leng = recv(sock, buffer, resp_leng, 0);
    if (resp_leng < 1)
    {
        if ((resp_leng == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
            continue;

        die_with_error("send() failed"); 
    }

    response += string(buffer, resp_leng);

    if (!http_leng_needed)
        http_leng -= resp_leng;
} 
while ((http_leng_needed) || (http_leng > 0));

::QueryPerformanceCounter(&httpResponseGot); 

//display response 
cout << response << endl; 

// Display the HTTP duration 
httpDuration = (double)(httpResponseGot.QuadPart - httpRequestSent.QuadPart) / (double)frequency.QuadPart; 
printf("The HTTP duration is %lf\n", httpDuration); 

With this said, the "correct" way to handle HTTP in general is to read the inbound data line by line, rather than buffer by buffer, until you encounter the end of the response headers, then you can read the rest of the data buffer by buffer based on the data length indicated by the headers.

Answer 2

Here is a second answer with more dynamic buffer handling and more error checking:

void send_data(SOCKET sock, void *data, unsigned int data_len)
{
    unsigned char *ptr = (unsigned char*) data;

    while (data_len > 0)
    {
        int num_to_send = (int) std::min(1024*1024, data_len);

        int num_sent = send(sock, ptr, num_to_send, 0);
        if (num_sent < 0)
        {
            if ((num_sent == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
                continue;

            die_with_error("send() failed");
        } 

        if (num_sent == 0)
            die_with_error("socket disconnected");

        ptr += num_sent;
        data_len -= num_sent;
    } 
}

unsigned int recv_data(SOCKET sock, void *data, unsigned int data_len, bool error_on_disconnect = true)
{
    unsigned char *ptr = (unsigned char*) data;
    unsigned int total = 0;

    while (data_len > 0)
    {
        int num_to_recv = (int) std::min(1024*1024, data_len);

        int num_recvd = recv(sock, ptr, num_to_recv, 0);
        if (num_recvd < 0)
        {
            if ((num_recvd == SOCKET_ERROR) && (WSAGetLastError() == WSAEWOULDBLOCK))
                continue;

            die_with_error("recv() failed");
        }

        if (num_recvd == 0)
        {
            if (error_on_disconnect)
                die_with_error("socket disconnected");

            break;
        }

        ptr += num_recvd;
        datalen -= num_recvd;
        total += num_recvd;
    }
    while (true);

    return total;
}

std::string recv_line(SOCKET sock)
{
    std::string line;
    char c;

    do
    {
        recv_data(sock, &c, 1);

        if (c == '\r')
        {
            recv_data(sock, &c, 1);

            if (c == '\n')
                break;

            line += '\r';
        }

        else if (c == '\n')
            break;

        line += c;
    }

    return line;
}

void recv_headers(SOCKET sock, std::vector<std::string> *hdrs)
{
    do
    {  
        std::string line = recv_line(sock);
        if (line.length() == 0)
          return;

        if (hdrs)
            hdrs->push_back(line);
    }
    while (true);
}

unsigned int recv_chunk_size(SOCKET sock)
{
    std::string line = recv_line(sock);
    size_t pos = line.find(";");
    if (pos != std::string::npos)
      line.erase(pos);
    char *endptr;
    unsigned int value = strtoul(line.c_str(), &endptr, 16);
    if (*endptr != '\0')
        die_with_error("bad Chunk Size received");
    return value;
}

std::string find_header(const std::vector<std::string> &hrds, const std::string &hdr_name)
{
    std::string value;

    for(size_t i = 0; i < hdrs.size(); ++i)
    {
        const std::string hdr = hdrs[i];

        size_t pos = hdr.find(":");
        if (pos != std::string::npos)
        {
            if (hdr.compare(0, pos-1, name) == 0)
            {
                pos = hdr.find_first_not_of(" ", pos+1);
                if (pos != std::string::npos)
                    return hdr.substr(pos);

                break;
            }
        }
    }

    return "";
}

{
    // send request ...

    std::string request = ...;

    send_data(sock, request.c_str(), request.length());

    // Record the time of httpRequestSent
    ::QueryPerformanceCounter(&httpRequestSent);
    ::QueryPerformanceFrequency(&frequency);  

    // get response ...

    std::vector<std::string> resp_headers;
    std::vector<unsigned char> resp_data;

    recv_headers(sock, &resp_headers);

    std::string transfer_encoding = find_header(resp_headers, "Transfer-Encoding");
    if (transfer_encoding.find("chunked") != std::string::npos)
    {
        unsigned int chunk_len = recv_chunk_size(sock);
        while (chunk_len != 0)
        {
            size_t offset = resp_data.size();
            resp_data.resize(offset + chunk_len);
            recv_data(sock, &resp_data[offset], chunk_len);

            recv_line(sock);
            chunk_len = recv_chunk_size(sock);
        }

        recv_headers(sock, NULL);
    }
    else
    {
        std::string content_length = find_header(resp_headers, "Content-Length");
        if (content_length.length() != 0)
        {
            char *endptr;
            unsigned int content_length_value = strtoul(content_length.c_str(), &endptr, 10);

            if (*endptr != '\0')
                die_with_error("bad Content-Length value received");

            if (content_length_value > 0)
            {
                resp_data.resize(content_length_value);
                recv_data(sock, &resp_data[0], content_length_value);
            }
        }
        else
        {
            unsigned char buffer[BUFFERSIZE];
            do
            {
                unsigned int buffer_len = recv_data(sock, buffer, BUFFERSIZE, false);
                if (buffer_len == 0)
                    break;

                size_t offset = resp_data.size();
                resp_data.resize(offset + buffer_len);
                memcpy(&resp_data[offset], buffer, buffer_len);
            }
            while (true)
        }
    }

    ::QueryPerformanceCounter(&httpResponseGot);  

    // process resp_data as needed
    // may be compressed, encoded, etc...

    // Display the HTTP duration  
    httpDuration = (double)(httpResponseGot.QuadPart - httpRequestSent.QuadPart) / (double)frequency.QuadPart;  
    printf("The HTTP duration is %lf\n", httpDuration); 
}