HttpWebRequest in .NET - Logging into a webpage

Question

Hi - Simple question ahead!

STATUS:

I've come this far I've implemented a solution where I get a cookie with a session, but it still doesn't work. In the sniffer tool, I can see the difference between my application and when using the real website as this:

REAL: ASPSESSIONIDSCRAASDB=EFFBFPEAKOBJGLAPNABPLLCB; passes=15; ChatCannel=1

PROGRAM: ASPSESSIONIDSCRAASDB=KPGBFPEAHNDLENDEOAEELMPJ

The program acts like it's not logged in, even though I've stored the session as a cookie..

---

I'm trying to create a small program, helping me playing a niche Internet game (just calculations and stuff)..

Anyway - I need to log in! The login system is based on sessions ..

So... I've tried this:

string url = "http://server1.online-trucker.dk";

HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.Method = "POST";
req.ContentType = "application/x-www-form-urlencoded";

StreamWriter post = new StreamWriter(req.GetRequestStream());
post.Write("Username=MyUsername&Password=MyPassword");
post.Close();

HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
resp.GetResponseStream();
string kage = GetContentFromStream(resp.GetResponseStream()); 

Being a little naive, I expect "kage" should contain the response after clicking the "Log in" button, and my little crawler should have logged in.

The HTML I'm playing with:

       Brugernavn:
   <input type="text" name="Username" size="10" STYLE="font-size: 10px; background-color: #CCCCCC; border: 1px solid #666666;">
   Kode:<input type="password" name="Password" size="10" STYLE="font-size: 10px; background-color: #CCCCCC; border: 1px solid #666666;">
   <input type="submit" name="Logind" value="Logind" STYLE="font-size: 10px; background-color: #CCCCCC; border: 1px solid #666666;"> 

But if you have just a little knowledge about WebRequest, I'm sure you by now will laugh ! :-)

What I want to do:

• Put in username/password
• Click "Log ind"
• Be able to play in the whole domain of the site, with the now active session

I really hope someone will help!

Answer 1

A lot of these sites do their best to deter automated logins much like you're trying to do here. You need to make sure you're sending the exact data they're expecting. One way I typically do this is to run a packet sniffer and inspect the HTTP packets that fly back and forth when I'm logging in the normal way. Once I have a baseline to work with it's just a matter of getting the code to mimic the behaviors.

A good free packet sniffing app is WireShark. http://www.wireshark.org

Answer 2

Sessions are usually kept with cookies so you need to assign a cookie container to your request, which can be reused. So assuming your login code with query parameters are correct the following should help you.

var cookieJar = new CookieContainer();
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.CookieContainer = cookieJar;
//...make req...
req = (HttpWebRequest)WebRequest.Create(url); // request to the second page
req.CookieContainer = cookieJar; // pass inn the same cookie container

I've included a download wrapper class which keeps track of cookies, supports gzip and keeps track of the correct encoding of the page. Use it like this:

var dl = new Downloader("Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)")
var pageOne = dl.GetPage( "http://www.cnn.com/", null );
var pageTwo = dl.GetPage( "http://edition.cnn.com/2010/WORLD/europe/10/02/missing.balloonists/index.html", dl.Url);


public class Downloader
{
    static CookieContainer _cookieJar = new CookieContainer();
    private readonly string _userAgent;

    public Encoding Encoding { get; set; }
    public WebHeaderCollection Headers { get; set; }
    public Uri Url { get; set; }

    public static void ClearBag()
    {
        _cookieJar = new CookieContainer();
    }

    public Downloader(string userAgent)
    {
        Encoding = Encoding.GetEncoding("ISO-8859-1");
        _userAgent = userAgent;
    }

    public string GetPage(string url, string referer)
    {
        HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
        request.CookieContainer = _cookieJar;

        if (!string.IsNullOrEmpty(referer))
            request.Referer = referer;
        if (!string.IsNullOrEmpty(_userAgent))
            request.UserAgent = _userAgent;

        request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
        request.Headers.Add("Cache-Control", "no-cache");

        using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
        {
            Headers = response.Headers;
            Url = response.ResponseUri;
            return ProcessContent(response);
        }
    }

    private string ProcessContent(HttpWebResponse response)
    {
        SetEncodingFromHeader(response);

        Stream s = response.GetResponseStream();
        if (response.ContentEncoding.ToLower().Contains("gzip"))
            s = new GZipStream(s, CompressionMode.Decompress);
        else if (response.ContentEncoding.ToLower().Contains("deflate"))
            s = new DeflateStream(s, CompressionMode.Decompress);  

        MemoryStream memStream = new MemoryStream();
        int bytesRead;
        byte[] buffer = new byte[0x1000];
        for (bytesRead = s.Read(buffer, 0, buffer.Length); bytesRead > 0; bytesRead = s.Read(buffer, 0, buffer.Length))
        {
            memStream.Write(buffer, 0, bytesRead);
        }
        s.Close();
        string html;
        memStream.Position = 0;
        using (StreamReader r = new StreamReader(memStream, Encoding))
        {
            html = r.ReadToEnd().Trim();
            html = CheckMetaCharSetAndReEncode(memStream, html);
        }            
        return html;
    }

    private void SetEncodingFromHeader(HttpWebResponse response)
    {
        string charset = null;
        if (string.IsNullOrEmpty(response.CharacterSet))
        {
            Match m = Regex.Match(response.ContentType, @";\s*charset\s*=\s*(?<charset>.*)", RegexOptions.IgnoreCase);
            if (m.Success)
            {
                charset = m.Groups["charset"].Value.Trim(new[] { '\'', '"' });
            }
        }
        else
        {
            charset = response.CharacterSet;
        }
        if (!string.IsNullOrEmpty(charset))
        {
            try
            {
                Encoding = Encoding.GetEncoding(charset);
            }
            catch (ArgumentException)
            {
            }
        }
    }

    private string CheckMetaCharSetAndReEncode(Stream memStream, string html)
    {
        Match m = new Regex(@"<meta\s+.*?charset\s*=\s*(?<charset>[A-Za-z0-9_-]+)", RegexOptions.Singleline | RegexOptions.IgnoreCase).Match(html);
        if (m.Success)
        {
            string charset = m.Groups["charset"].Value.ToLower();
            if ((charset == "unicode") || (charset == "utf-16"))
            {
                charset = "utf-8";
            }

            try
            {
                Encoding metaEncoding = Encoding.GetEncoding(charset);
                if (Encoding != metaEncoding)
                {
                    memStream.Position = 0L;
                    StreamReader recodeReader = new StreamReader(memStream, metaEncoding);
                    html = recodeReader.ReadToEnd().Trim();
                    recodeReader.Close();
                }
            }
            catch (ArgumentException)
            {
            }
        }
        return html;
    }
}