The remote server returned an error: (401) Unauthorized.

Question

I am extracting gmail contacts using opencontactsnet. I get The remote server returned an error: (401) Unauthorized. when executing this line

HttpWebResponse exportResponse = ( HttpWebResponse ) contactsRequest.GetResponse();

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Data;
using System.Net;
using System.Web.UI;
using System.Web.UI.WebControls;
using OpenContactsNet;

public partial class _Default : System.Web.UI.Page 
{
    protected void Page_Load(object sender, EventArgs e)
    {
        GmailExtract gm = new GmailExtract();
        NetworkCredential nw = new NetworkCredential("chendur.pandiya","**");
        MailContactList mc;
        if (gm.Extract(nw, out mc))
        {

        }
        else
        {

        }
     }
}

Here is the complete GmailExtract class,

using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;
using Utilities.Web;

namespace OpenContactsNet
{
    public class GmailExtract : IMailContactExtract
    {
        private const string ContinueUrl = "http://mail.google.com/mail?ui=html&zy=l";
        private const string ExportUrl = "https://mail.google.com/mail/contacts/data/export?exportType=ALL&groupToExport=&out=GMAIL_CSV";
        private const string LoginRefererUrl = "https://www.google.com/accounts/ServiceLogin?service=mail&passive=true&rm=false&continue=http%3A%2F%2Fmail.google.com%2Fmail%2F%3Fui%3Dhtml%26zy%3Dl&ltmpl=default&ltmplcache=2";
        private const string LoginUrl = "https://www.google.com/accounts/ServiceLoginAuth?service=mail";
        private const string UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; EmbeddedWB 14.52 from: http://www.bsalsa.com/ EmbeddedWB 14.52; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1; .NET CLR 1.0.3705; .NET CLR 3.0.04506.30)";

        #region IMailContactExtract Members

        public bool Extract( NetworkCredential credential, out MailContactList list )
        {
            bool result = false;
            list = new MailContactList();

            try
            {
                CookieCollection cookies = new CookieCollection();

                // Prepare login form data
                HttpValueCollection loginFormValues = new HttpValueCollection();
                loginFormValues[ "ltmpl" ] = "default";
                loginFormValues[ "ltmplcache" ] = "2";
                loginFormValues[ "continue" ] = ContinueUrl;
                loginFormValues[ "service" ] = "mail";
                loginFormValues[ "rm" ] = "false";
                loginFormValues[ "hl" ] = "en";
                loginFormValues[ "Email" ] = credential.UserName;
                loginFormValues[ "Passwd" ] = credential.Password;
                loginFormValues[ "PersistentCookie" ] = "true";
                loginFormValues[ "rmShown" ] = "1";
                loginFormValues[ "null" ] = "Sign In";

                // Convert to bytes
                byte[] loginPostData = Encoding.UTF8.GetBytes( loginFormValues.ToString( true ) );

                HttpWebRequest loginRequest = ( HttpWebRequest ) WebRequest.Create( LoginUrl );
                loginRequest.Method = "POST";
                loginRequest.UserAgent = UserAgent;
                loginRequest.Referer = LoginRefererUrl;
                loginRequest.ContentType = "application/x-www-form-urlencoded";
                loginRequest.ContentLength = loginPostData.Length;
                loginRequest.AllowAutoRedirect = false;

                // Create cookie container
                loginRequest.CookieContainer = new CookieContainer();

                // Add post data to request
                Stream stream;
                using ( stream = loginRequest.GetRequestStream() )
                {
                    stream.Write( loginPostData, 0, loginPostData.Length );
                }

                HttpWebResponse loginResponse = ( HttpWebResponse ) loginRequest.GetResponse();

                cookies.Add( loginResponse.Cookies );

                // Create request to export Google CSV page
                HttpWebRequest contactsRequest = ( HttpWebRequest ) WebRequest.Create( ExportUrl );
                contactsRequest.Method = "GET";
                contactsRequest.UserAgent = UserAgent;
                contactsRequest.Referer = loginResponse.ResponseUri.ToString();

                // use cookie gotten from login page
                contactsRequest.CookieContainer = new CookieContainer();
                foreach ( Cookie cookie in cookies )
                {
                    contactsRequest.CookieContainer.Add( cookie );
                }

                HttpWebResponse exportResponse = ( HttpWebResponse ) contactsRequest.GetResponse();

                // Read data from response stream
                string csvData;
                using ( Stream responseStream = exportResponse.GetResponseStream() )
                {
                    using ( StreamReader streamRead = new StreamReader( responseStream ) )
                    {
                        csvData = streamRead.ReadToEnd();
                    }
                }

                // parse google csv
                string[] lines = csvData.Split( '\n' );
                foreach ( string line in lines )
                {
                    string[] values = line.Split( ',' );
                    if ( values.Length < 2 )
                    {
                        continue;
                    }

                    MailContact mailContact = new MailContact();
                    mailContact.Email = values[ 1 ];
                    mailContact.Name = values[ 0 ];
                    list.Add( mailContact );
                }

                result = true;
            }
            catch (Exception e)
            {
                throw e;
            }

            return result;
        }

        #endregion
    }
}

Answer 1

I am not familiar with google's api nor do I know C#. However check the URL you are sending your form to. Looks like your authentication works. Otherwise you would have recieved a 403. 401 means that you (as an authenticated user) are not allowed to access the requested resource.

Answer 2

10.4.2 401 Unauthorized

The request requires user authentication. The response MUST include a WWW-Authenticate header field (section 14.47) containing a challenge applicable to the requested resource. The client MAY repeat the request with a suitable Authorization header field (section 14.8). If the request already included Authorization credentials, then the 401 response indicates that authorization has been refused for those credentials. If the 401 response contains the same challenge as the prior response, and the user agent has already attempted authentication at least once, then the user SHOULD be presented the entity that was given in the response, since that entity might include relevant diagnostic information.

Is the LoginRefererUrl correctly formed? As that address isn't found on google servers however the correctly formed version signs me straight into my mail box.

Answer 3

I ran into the same problem. I had to change the GmailExtract class in opencontactsnet. Below is the new code which works for me.

public class GmailExtract : IMailContactExtract
    {
        private const string ContinueUrl = "https://mail.google.com/mail/?"; 
        private const string ExportUrl = "https://mail.google.com/mail/contacts/data/export?exportType=ALL&groupToExport=&out=GMAIL_CSV";
        private const string LoginRefererUrl = "https://www.google.com/accounts/ServiceLogin?service=mail&passive=true&rm=false&continue=http%3A%2F%2Fmail.google.com%2Fmail%2F%3Fui%3Dhtml%26zy%3Dl&ltmpl=default&ltmplcache=2";
        private const string LoginUrl = "https://www.google.com/accounts/ServiceLoginAuth?service=mail";
        private const string UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)";

        #region IMailContactExtract Members

        public bool Extract( NetworkCredential credential, out MailContactList list)
        {
            bool result = false;
            list = new MailContactList();

            DateTime jsStartDate = new DateTime(1970, 1, 1);
            TimeSpan endTs = DateTime.Now.Subtract(jsStartDate);
            TimeSpan startTs = DateTime.Now.AddMinutes(-2).Subtract(jsStartDate);

            try
            {
                CookieCollection cookies = new CookieCollection();

                // Prepare login form data
                HttpValueCollection loginFormValues = new HttpValueCollection();
                loginFormValues["Email"] = credential.UserName;
                loginFormValues["Passwd"] = credential.Password;
                loginFormValues["asts"] = "";
                loginFormValues["continue"] = ContinueUrl;
                loginFormValues["dsh"] = "1461574034599761425";
                loginFormValues["hl"] = "en";
                loginFormValues["ltmpl"] = "default";
                loginFormValues["ltmplcache"] = "2";
                loginFormValues["rm"] = "false";
                loginFormValues["rmShown"] = "1";
                loginFormValues["service"] = "mail";
                loginFormValues["signIn"] = "Sign In";
                loginFormValues["scc"] = "1";
                loginFormValues["ss"] = "1";
                loginFormValues["GALX"] = "rBTUs4OAJBI";
                loginFormValues["ltmpl"] = "default";
                loginFormValues["ltmpl"] = "default";

                // Convert to bytes
                byte[] loginPostData = Encoding.UTF8.GetBytes( loginFormValues.ToString( true ) );

                HttpWebRequest loginRequest = ( HttpWebRequest ) WebRequest.Create( LoginUrl );
                loginRequest.Method = "POST";
                loginRequest.UserAgent = UserAgent;
                loginRequest.Referer = LoginRefererUrl;
                loginRequest.ContentType = "application/x-www-form-urlencoded";
                loginRequest.ContentLength = loginPostData.Length;
                loginRequest.AllowAutoRedirect = false;

                // Create cookie container
                loginRequest.CookieContainer = new CookieContainer();
                loginRequest.CookieContainer.Add(new Cookie("GMAIL_LOGIN", "T" + startTs.Milliseconds.ToString() + "/" + startTs.Milliseconds.ToString() + "/" + endTs.Milliseconds.ToString(), "/", ".google.com"));
                loginRequest.CookieContainer.Add(new Cookie("GALX", "rBTUs4OAJBI", "/accounts", ".google.com"));

                // Add post data to request
                Stream stream;
                using ( stream = loginRequest.GetRequestStream())
                {
                    stream.Write( loginPostData, 0, loginPostData.Length);
                }

                HttpWebResponse loginResponse = ( HttpWebResponse ) loginRequest.GetResponse();

                cookies.Add( loginResponse.Cookies );

                // Create request to export Google CSV page
                HttpWebRequest contactsRequest = ( HttpWebRequest ) WebRequest.Create( ExportUrl );
                contactsRequest.Method = "GET";
                contactsRequest.UserAgent = UserAgent;
                contactsRequest.Referer = loginResponse.ResponseUri.ToString();

                // use cookie gotten from login page
                contactsRequest.CookieContainer = new CookieContainer();
                foreach ( Cookie cookie in cookies )
                {
                    contactsRequest.CookieContainer.Add( cookie );
                }

                HttpWebResponse exportResponse = ( HttpWebResponse ) contactsRequest.GetResponse();

                // Read data from response stream
                string csvData;
                using ( Stream responseStream = exportResponse.GetResponseStream())
                {
                    using ( StreamReader streamRead = new StreamReader( responseStream ) )
                    {
                        csvData = streamRead.ReadToEnd();
                    }
                }

                // parse google csv
                string[] lines = csvData.Split( '\n' );
                foreach ( string line in lines )
                {
                    string[] values = line.Split( ',' );
                    if ( values.Length < 2 )
                    {
                        continue;
                    }

                    MailContact mailContact = new MailContact();
                    mailContact.Email = values[ 28 ];
                    mailContact.Name = values[ 0 ];
                    if (mailContact.Email.Trim().Length > 0)
                    {

                         list.Add(mailContact);

                    }
                }

                result = true;
            }
            catch
            {
            }

            return result;
        }

        #endregion
    }