Poor boost.ASIO performance.

Question

I have a very simple server/client performance test using boost::asio on Windows and it seems to be performing really poorly. I'm hoping that I'm just using the library incorrectly and would appreciate any advice.

I have a session class that writes a message-length and then writes a message, and then waits to read a message-length and then read a message, and keeps doing this over and over again nonstop. When I run it locally on my own computer I get blazingly fast performance, however; when I run a server on one computer and a client on another computer, even on the same network, the performance slows down, taking as much as 1 second for a read/write operation to occur.

The server source code file is as follows:

#include <cstdlib>
#include <iostream>
#include <boost/asio.hpp>
#include <boost/bind.hpp>

using namespace boost;
using namespace boost::asio;
using namespace boost::asio::ip;
using namespace std;

class Session {
  public:

    Session(io_service& ioService)
      : m_socket(ioService) {}

    tcp::socket& GetSocket() {
      return m_socket;
    }

    void StartRead() {
      m_messageSizeIterator = reinterpret_cast<char*>(&m_messageSize);
      async_read(m_socket, buffer(m_messageSizeIterator, sizeof(m_messageSize)),
        bind(&Session::HandleSizeRead, this, placeholders::error,
        placeholders::bytes_transferred));
    }

    void StartWrite(const char* message, int messageSize) {
      m_messageSize = messageSize;
      m_message = new char[m_messageSize];
      memcpy(m_message, message, m_messageSize);
      async_write(m_socket, buffer(&m_messageSize, sizeof(int)),
        bind(&Session::HandleSizeWritten, this, placeholders::error));
    }

    void HandleSizeRead(const system::error_code& error,
        size_t bytes_transferred) {
      if(!error) {
        m_message = new char[m_messageSize];
        async_read(m_socket, buffer(m_message, m_messageSize),
          bind(&Session::HandleMessageRead, this, placeholders::error,
          placeholders::bytes_transferred));
      } else {
        delete this;
      }
    }

    void HandleMessageRead(const system::error_code& error,
        size_t bytes_transferred) {
      if(!error) {
        cout << string(m_message, m_messageSize) << endl;
        async_write(m_socket, buffer(&m_messageSize, sizeof(int)),
          bind(&Session::HandleSizeWritten, this, placeholders::error));
      } else {
        delete this;
      }
    }

    void HandleSizeWritten(const system::error_code& error) {
      if(!error) {
        async_write(m_socket, buffer(m_message, m_messageSize),
          bind(&Session::HandleMessageWritten, this, placeholders::error));
      } else {
        delete this;
      }
    }

    void HandleMessageWritten(const system::error_code& error) {
      if(!error) {
        delete m_message;
        m_messageSizeIterator = reinterpret_cast<char*>(&m_messageSize);
        async_read(m_socket, buffer(m_messageSizeIterator,
          sizeof(m_messageSize)), bind(&Session::HandleSizeRead, this,
          placeholders::error, placeholders::bytes_transferred));
      } else {
        delete this;
      }
    }

  private:
    tcp::socket m_socket;
    int m_messageSize;
    char* m_messageSizeIterator;
    char* m_message;
};

class Server {
  public:

    Server(io_service& ioService, short port)
        : m_ioService(ioService),
          m_acceptor(ioService, tcp::endpoint(tcp::v4(), port)) {
      Session* new_session = new Session(m_ioService);
      m_acceptor.async_accept(new_session->GetSocket(), bind(&Server::HandleAccept,
        this, new_session,asio::placeholders::error));
    }

    void HandleAccept(Session* new_session, const system::error_code& error) {
      if(!error) {
        new_session->StartRead();
        new_session = new Session(m_ioService);
        m_acceptor.async_accept(new_session->GetSocket(), bind(
          &Server::HandleAccept, this, new_session, placeholders::error));
      } else {
        delete new_session;
      }
    }

  private:
    io_service& m_ioService;
    tcp::acceptor m_acceptor;
};

int main(int argc, char* argv[]) {
  try {
    if(argc != 2) {
      cerr << "Usage: server <port>\n";
      return 1;
    }
    io_service io_service;
    Server s(io_service, atoi(argv[1]));
    io_service.run();
  } catch(std::exception& e) {
    cerr << "Exception: " << e.what() << "\n";
  }
  return 0;
}

And the client code is as follows:

#include <cstdlib>
#include <cstring>
#include <iostream>
#include <boost/bind.hpp>
#include <boost/asio.hpp>

using namespace boost;
using namespace boost::asio;
using namespace boost::asio::ip;
using namespace std;

class Session {
  public:

    Session(io_service& ioService)
      : m_socket(ioService) {}

    tcp::socket& GetSocket() {
      return m_socket;
    }

    void StartRead() {
      m_messageSizeIterator = reinterpret_cast<char*>(&m_messageSize);
      async_read(m_socket, buffer(m_messageSizeIterator, sizeof(m_messageSize)),
        bind(&Session::HandleSizeRead, this, placeholders::error,
        placeholders::bytes_transferred));
    }

    void StartWrite(const char* message, int messageSize) {
      m_messageSize = messageSize;
      m_message = new char[m_messageSize];
      memcpy(m_message, message, m_messageSize);
      async_write(m_socket, buffer(&m_messageSize, sizeof(int)),
        bind(&Session::HandleSizeWritten, this, placeholders::error));
    }

    void HandleSizeRead(const system::error_code& error,
        size_t bytes_transferred) {
      if(!error) {
        m_message = new char[m_messageSize];
        async_read(m_socket, buffer(m_message, m_messageSize),
          bind(&Session::HandleMessageRead, this, placeholders::error,
          placeholders::bytes_transferred));
      } else {
        delete this;
      }
    }

    void HandleMessageRead(const system::error_code& error,
        size_t bytes_transferred) {
      if(!error) {
        cout << string(m_message, m_messageSize) << endl;
        async_write(m_socket, buffer(&m_messageSize, sizeof(int)),
          bind(&Session::HandleSizeWritten, this, placeholders::error));
      } else {
        delete this;
      }
    }

    void HandleSizeWritten(const system::error_code& error) {
      if(!error) {
        async_write(m_socket, buffer(m_message, m_messageSize),
          bind(&Session::HandleMessageWritten, this, placeholders::error));
      } else {
        delete this;
      }
    }

    void HandleMessageWritten(const system::error_code& error) {
      if(!error) {
        delete m_message;
        m_messageSizeIterator = reinterpret_cast<char*>(&m_messageSize);
        async_read(m_socket, buffer(m_messageSizeIterator,
          sizeof(m_messageSize)), bind(&Session::HandleSizeRead, this,
          placeholders::error, placeholders::bytes_transferred));
      } else {
        delete this;
      }
    }

  private:
    tcp::socket m_socket;
    int m_messageSize;
    char* m_messageSizeIterator;
    char* m_message;
};

int main(int argc, char* argv[]) {
  try {
    if(argc != 3) {
      cerr << "Usage: client <host> <port>\n";
      return 1;
    }
    io_service io_service;
    tcp::resolver resolver(io_service);
    tcp::resolver::query query(tcp::v4(), argv[1], argv[2]);
    tcp::resolver::iterator iterator = resolver.resolve(query);
    Session session(io_service);
    tcp::socket& s = session.GetSocket();
    s.connect(*iterator);
    cout << "Enter message: ";
    const int MAX_LENGTH = 1024;
    char request[MAX_LENGTH];
    cin.getline(request, MAX_LENGTH);
    int requestLength = strlen(request);
    session.StartWrite(request, requestLength);
    io_service.run();
  } catch (std::exception& e) {
    cerr << "Exception: " << e.what() << "\n";
  }
  return 0;
}

Any help would be appreciated, thanks.

Answer 1

For my purposes, sending really really small messages and wanting virtual real time replies, disabling Nagle's algorithm turned out to be the cause of the poor performance.

Answer 2

You must turn off the Nagle algorithm. Call:

m_socket.set_option(tcp::no_delay(true));

Where appropriate for your code.

Answer 3

I'd also look at using a message pool. you're new'ing and delete'ing m_message upon every successful async call.

Also have a go at getting the asio benchmark running on your system, the numbers will show the near best results you could ever get on your system using asio.