C# HTTPWebRequest Multi-threading

Question

Hi all,

I am new to threading. I am trying to send HTTP Web Request using multi threading, I am not able to acheive what I need. My requirement is to send request to thousands of same or different websites and parse the response i get it from httpwebrequest. In the below code, i am sending 2 simulteaneous threads, I am looking for ten simultaneously threads.

namespace threading
{
public partial class Form1 : Form
{
    delegate string UrlFetcher(string url);

    private void button1_Click(object sender, EventArgs e)
    {
        int i = 1;
        UrlFetcher u = new UrlFetcher(Fetch);
        UrlFetcher u = new UrlFetcher(Fetch1);
        string pageURL = "http://www.google.com";

        while (i <= 1000)
        {
            u.BeginInvoke(pageURL, new AsyncCallback(AfterFetch), "this is state");
            i++;
            u.BeginInvoke(pageURL, new AsyncCallback(AfterFetch1), "this is state");
            i++;
            Thread.Sleep(5);
        }
    }

    static string Fetch(string pageURL)
    {
        HttpWebRequest WebReq = (HttpWebRequest)WebRequest.Create(pageURL);
        WebReq.Method = "GET";
        HttpWebResponse WebResp = (HttpWebResponse)WebReq.GetResponse();
        Stream Answer = WebResp.GetResponseStream();
        StreamReader _Answer = new StreamReader(Answer);
        string myString = _Answer.ReadToEnd();
        return myString;
    }

    void AfterFetch(IAsyncResult result)
    {
        string a;

        AsyncResult async = (AsyncResult)result;
        UrlFetcher fetcher = (UrlFetcher)async.AsyncDelegate;
        a = fetcher.EndInvoke(result).ToString();

        Regex regx = new Regex(@"<td>([A-Za-z0-9\-]+)\.(com|net)</td>", RegexOptions.IgnoreCase);
        MatchCollection mactches = regx.Matches(a);
        foreach (Match match in mactches)
        {
            string pattern = @"<(.|\n)*?>";
            string r = Regex.Replace(match.Value, pattern, string.Empty);
            textBox3.AppendText(r);
        }
    }

    static string Fetch1(string pageURL)
    {
        HttpWebRequest WebReq = (HttpWebRequest)WebRequest.Create(pageURL);
        WebReq.Method = "GET";
        HttpWebResponse WebResp = (HttpWebResponse)WebReq.GetResponse();
        Stream Answer = WebResp.GetResponseStream();
        StreamReader _Answer = new StreamReader(Answer);
        string myString = _Answer.ReadToEnd();
        return myString;
    }

    void AfterFetch1(IAsyncResult result)
    {
        string a;

        AsyncResult async = (AsyncResult)result;
        UrlFetcher fetcher = (UrlFetcher)async.AsyncDelegate;
        a = fetcher.EndInvoke(result).ToString();

        Regex regx = new Regex(@"<td>([A-Za-z0-9\-]+)\.(com|net)</td>", RegexOptions.IgnoreCase);
        MatchCollection mactches = regx.Matches(a);
        foreach (Match match in mactches)
        {
            string pattern = @"<(.|\n)*?>";
            string r = Regex.Replace(match.Value, pattern, string.Empty);
            textBox3.AppendText(r);
        }
    }
}
}

If anyone would correct the above code, it is really appreciated.

Thanks

Answer 1

I would say

• Abolish your delegate
• Set up a WebRequest in the loop
• Use the async version of Getting the response (Begin/End)GetResponse
• Keep your async callback reentrant (independent of any instance state) and make it use the result form the "End" call and any state you pass in (e.g. the WebRequest itself)

That should more or less work

Answer 2

How to end the response, Could anyone please add it to the above code, so that it works fine for 10 threads simultaneously and sending 1000 of request.