Libcurl via threads! Problem

Question

Hello

I am coding program on c# (C#, & object oriented programming newbie) I am trying to download pages in multiple threads. There are no problems when running only 1 thread. Problems starts when I run multiple.

It seems like all of the threads save the downloaded info in the same variable SockBuff, but I have no ideas what to do to solve this problem.

Suggestions ?

Here is the code

Program starts when I click button.

namespace WindowsFormsApplication1
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        void TestaThread_1()
        {
            Random RandomNumber = new Random();
            int rand = RandomNumber.Next(99);

            HTTP cURL = new HTTP();
            cURL.CurlInit();
            String data = cURL.HTTPGet("http://google.com", "", "fails" + rand.ToString() + ".html");
            HTTP.save_file("fails" + rand.ToString()+ ".html", data);
        }

        private void button1_Click(object sender, EventArgs e)
        {
            Thread thread1 = new Thread(new ThreadStart(TestaThread_1));
            thread1.Start();

            Thread thread2 = new Thread(new ThreadStart(TestaThread_1));
            thread2.Start();

            Thread thread3 = new Thread(new ThreadStart(TestaThread_1));
            thread3.Start();

            Thread thread4 = new Thread(new ThreadStart(TestaThread_1));
            thread4.Start();

            Thread thread5 = new Thread(new ThreadStart(TestaThread_1));
            thread5.Start();
        }
    }

    class HTTP
    {
        public Easy easy;        
        public static string SockBuff;
        public string CookieFile = AppDomain.CurrentDomain.BaseDirectory + "cookie.txt";
        public string UserAgent = "Mozilla 5.0";
        public string Proxy = "";


        public void CurlInit()
        {
            Curl.GlobalInit((int)CURLinitFlag.CURL_GLOBAL_ALL);
        }


        public string HTTPGet(string URL, string Proxy, String FailaNosaukums)
        {
            easy = new Easy();
            SockBuff = "";

            try
            {
                Easy.WriteFunction wf = new Easy.WriteFunction(OnWriteData);

                easy.SetOpt(CURLoption.CURLOPT_URL, URL);
                easy.SetOpt(CURLoption.CURLOPT_TIMEOUT, "60");
                easy.SetOpt(CURLoption.CURLOPT_WRITEFUNCTION, wf);
                easy.SetOpt(CURLoption.CURLOPT_USERAGENT, UserAgent);
                easy.SetOpt(CURLoption.CURLOPT_COOKIEFILE, CookieFile);
                easy.SetOpt(CURLoption.CURLOPT_COOKIEJAR, CookieFile);
                easy.SetOpt(CURLoption.CURLOPT_FOLLOWLOCATION, true);

                if (URL.Contains("https"))
                {
                    easy.SetOpt(CURLoption.CURLOPT_SSL_VERIFYHOST, 1);
                    easy.SetOpt(CURLoption.CURLOPT_SSL_VERIFYPEER, 0);
                }

                if (Proxy != "")
                {
                    easy.SetOpt(CURLoption.CURLOPT_PROXY, Proxy);
                    easy.SetOpt(CURLoption.CURLOPT_PROXYTYPE, CURLproxyType.CURLPROXY_HTTP);
                }

                easy.Perform();
                easy.Cleanup();

            }
            catch
            {
                Console.WriteLine("Get Request Error");
            }

            return SockBuff;
        }


        public static Int32 OnWriteData(Byte[] buf, Int32 size, Int32 nmemb, Object extraData)
        {
            Random rand = new Random();
            int RandomNumber = rand.Next(1000);

            SockBuff = SockBuff + System.Text.Encoding.UTF8.GetString(buf);

            return size * nmemb;
        }

        static public void save_file(string file_name, string text_to_write)
        {
            StreamWriter MyStream = null;
            string MyString = text_to_write;

            try
            {
                MyStream = File.CreateText(file_name);
                MyStream.Write(MyString);
            }
            catch (IOException e)
            {
                Console.WriteLine(e);
            }
            catch (Exception e)
            {
                Console.WriteLine(e);
            }
            finally
            {
                if (MyStream != null)
                    MyStream.Close();
            }
        }
    }
}

Answer 1

The problem is that your SockBuff is static, so there's only one instance of that variable that's shared by all instances of the HTTP class. You can probably solve the problem by removing the static.

If you do that, you'll also have to make your OnWriteData method an instance method (i.e. not static).

You might also consider looking into using System.Net.WebClient and/or System.Net.HttpWebRequest classes instead of Libcurl.

Answer 2

I got another question.

Why can't I save image files using save_file function correctly? Are there encoding problems in the buff ?