Intermittent DownloadError Application Error 2 on Google App Engine

Question

We have two applications that are both running on Google App Engine. App1 makes requests to app2 as an authenticated user. The authentication works by requesting an authentication token from Google ClientLogin that is exchanged for a cookie. The cookie is then used for subsequent requests (as described here). App1 runs the following code:

class AuthConnection:

    def __init__(self):            
        self.cookie_jar = cookielib.CookieJar()    
        self.opener = urllib2.OpenerDirector()
        self.opener.add_handler(urllib2.ProxyHandler())
        self.opener.add_handler(urllib2.UnknownHandler())
        self.opener.add_handler(urllib2.HTTPHandler())
        self.opener.add_handler(urllib2.HTTPRedirectHandler())
        self.opener.add_handler(urllib2.HTTPDefaultErrorHandler())
        self.opener.add_handler(urllib2.HTTPSHandler())
        self.opener.add_handler(urllib2.HTTPErrorProcessor())
        self.opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; ' +\
                                       'Windows NT 6.1; en-US; rv:1.9.1.2) ' +\
                                       'Gecko/20090729 Firefox/3.5.2 ' +\
                                       '(.NET CLR 3.5.30729)'
                        }

    def fetch(self, url, method, payload=None):
        self.__updateJar(url)
        request = urllib2.Request(url)
        request.get_method = lambda: method
        for key, value in self.headers.iteritems():
            request.add_header(key, value)
        response = self.opener.open(request)
        return response.read()

    def __updateJar(self, url):

        cache = memcache.Client()
        cookie = cache.get('auth_cookie')

        if cookie:
            self.cookie_jar.set_cookie(cookie)
        else:
            cookie = self.__retrieveCookie(url=url)
            cache.set('auth_cookie', cookie, 5000)  


    def __getCookie(self, url):
        auth_url = 'https://www.google.com/accounts/ClientLogin'
        auth_data = urllib.urlencode({'Email': USER_NAME,
                                      'Passwd': PASSPHRASE,
                                      'service': 'ah',
                                      'source':  'app1',
                                      'accountType': 'HOSTED_OR_GOOGLE' })
        auth_request = urllib2.Request(auth_url, data=auth_data)
        auth_response_body = self.opener.open(auth_request).read()
        auth_response_dict = dict(x.split('=') 
                for x in auth_response_body.split('\n') if x)
        cookie_args = {}
        cookie_args['continue'] = url
        cookie_args['auth'] = auth_response_dict['Auth']
        cookie_url = 'https://%s/_ah/login?%s' %\
                ('app2.appspot.com', (urllib.urlencode(cookie_args)))
        cookie_request = urllib2.Request(cookie_url)

        for key, value in self.headers.iteritems():
            cookie_request.add_header(key, value)

        try:
            self.opener.open(cookie_request)
        except:
            pass

        for cookie in self.cookie_jar:                         
            if cookie.domain == 'app2domain':
                return cookie

For 10-30% of the requests a DownloadError is raised:

Error fetching https://app2/Resource
Traceback (most recent call last):
  File "/base/data/home/apps/app1/5.344034030246386521/source/main/connection/authenticate.py", line 112, in fetch
    response = self.opener.open(request)
  File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 381, in open
    response = self._open(req, data)
  File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 399, in _open
    '_open', req)
  File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 360, in _call_chain
    result = func(*args)
  File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 1115, in https_open
    return self.do_open(httplib.HTTPSConnection, req)
  File "/base/python_runtime/python_dist/lib/python2.5/urllib2.py", line 1080, in do_open
    r = h.getresponse()
  File "/base/python_runtime/python_dist/lib/python2.5/httplib.py", line 197, in getresponse
    self._allow_truncated, self._follow_redirects)
  File "/base/data/home/apps/app1/5.344034030246386521/source/main/connection/monkeypatch_urlfetch_deadline.py", line 18, in new_fetch
    follow_redirects, deadline, *args, **kwargs)
  File "/base/python_runtime/python_lib/versions/1/google/appengine/api/urlfetch.py", line 241, in fetch
    return rpc.get_result()
  File "/base/python_runtime/python_lib/versions/1/google/appengine/api/apiproxy_stub_map.py", line 501, in get_result
    return self.__get_result_hook(self)
  File "/base/python_runtime/python_lib/versions/1/google/appengine/api/urlfetch.py", line 325, in _get_fetch_result
    raise DownloadError(str(err))
DownloadError: ApplicationError: 2 

The request logs for app2 (the "server") seem fine, as expected (according to the docs DownloadError is only raised if there was no valid HTTP response).

Why is the exception raised?

Answer 1

see this: http://bitbucket.org/guilin/gae-rproxy/src/tip/gae_rproxy/niceurllib.py

because of urllib and urllib2 default to handle http 302 code, and automatically redirect to what the server told it. But when redirect it does not contains the cookie which the server told it.

for example:

1. urllib2 request //server/login
2. server response 302, //server/profile , set-cookie : session-id:xxxx
3. urllib2 request //server/profile
4. server response not login error or 500 error cause there is no session-id found.
5. urllib2 throw error

so, there is no chance for you to set cookie.

self.opener.add_handler(urllib2.HTTPRedirectHandler())

I think you should remove this line and add your own HTTPRedirectHandler which neighter throw error nor automatically redirect , just return the http code and headers, so you have the chance to set cookie.