c# Socket not timing out

Question

I have the following c# code to query whois servers, occasionally I will hit a server and the socket will never time out the Receive (unfortunately if I hit the same server again all of a sudden the problem goes away). I have set the ReceiveTimeout value though, anyone know what is going wrong here?

string whoisServer = "whois.moniker.com";
Socket s = null;
IPAddress ipAddress;
ipAddress = Dns.GetHostEntry(whoisServer).AddressList[0];                

s = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
s.ReceiveTimeout = 10000;
s.SendTimeout = 10000;
s.Connect(new IPEndPoint(ipAddress, 43));
s.Send(Encoding.ASCII.GetBytes(domain + "\r\n"));
byte[] buffer = new byte[1024];
int recv = s.Receive(buffer);
while (recv > 0)
{
    ret += Encoding.ASCII.GetString(buffer, 0, recv);
    recv = s.Receive(buffer);
}
s.Shutdown(SocketShutdown.Both);
s.Close();