I use a combination of HtmlAgilityPack and fiddler to watch the traffic. With fiddler you can see the post that is being made to the server, and just replicate it. Then you can capture the cookie data to save the login session. From there its just normal navigating around the site.
A few things to watch out for, if you use a login to get the data there may be a cookie. If so you need to set it in the cookieContainer. This is a sample function that I use to post data to an url.
private string PostWebRequestContent(Uri uri, string postData)
{
CookieContainer cookies = LoggedInCookie();
foreach (Cookie c in cookies.GetCookies(uri))
{
int i = 0;
}
HttpWebRequest req = GetWebRequest(uri);
req.CookieContainer = cookies;
req.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-ms-application, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-ms-xbap, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-silverlight, application/vnd.xfdl; version=7.5.0.51, application/x-shockwave-flash, */*";
req.Method = "POST";
req.Headers.Add("Accept-Language: en-us");
//req.Headers.Add("Accept-Encoding: gzip,deflate");
//req.Headers.Add("Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7");
req.KeepAlive = true;
req.ContentType = "application/x-www-form-urlencoded";
StreamWriter requestWriter = new StreamWriter(req.GetRequestStream());
requestWriter.Write(postData);
requestWriter.Close();
string sid = "";
try
{
using (HttpWebResponse res = (HttpWebResponse)req.GetResponse())
{
using (StreamReader sr = new StreamReader(res.GetResponseStream()))
{
sid = sr.ReadToEnd().Trim();
}
}
}
catch (WebException e)
{
Console.WriteLine(e.Message);
}
return sid;
}
Cookie session function
private CookieContainer GetSessionCookie(Uri uri)
{
HttpWebRequest req = GetWebRequest(uri);
CookieContainer cookies = new CookieContainer();
req.CookieContainer = cookies;
req.GetResponse().Close();
return cookies;
}