Script access to WebGoat urls?

Question

I've been solving a couple of the WebGoat exampels for a uni-lab thing. In one of the exercises I tried to use a python script with urllib2 to do automated "tests" so I didnt manually have to used ascii(substr(first_name,3,1)) > 97 etc.

But I seem to get the same page eventhough I try different urls when using an urllib2 script aka f2, f3 and f4.html is all the basic first page and not the same page as if you access it in a browser:

import urllib2 
import urllib
import cookielib
import sys


myjar = cookielib.FileCookieJar("cookies.txt");

cookieHandler = urllib2.HTTPCookieProcessor(myjar)

password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
tlurl="http://localhost:8081/webgoat/attack"

password_mgr.add_password(None,tlurl,user="guest",passwd="guest")
authhandler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(cookieHandler, authhandler)
data = [('Connection','keep-alive'),('User-Agent', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; da; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7')]



def write_f_to_f(fname,f):
    f1 = open(fname,"w")
    f1.write(f.read())
    f1.close()

def gen_req(url,referer=None):
    req = urllib2.Request(url)
    print "url: %s"%url
    mydata = data
    if referer != None:
       mydata.append( ('Referer',referer) ) 
    [ req.add_header(k,v) for k,v in mydata ]
    return req


sys.stdout.flush()
url = "http://localhost:8081/webgoat/attack"
req = gen_req(url)
f = opener.open(req)
write_f_to_f("f1.html",f)
f.close()

params = urllib.urlencode({'start':'Start WebGoat'})
qs = urllib.urlencode( {'Screen':107, 'menu':1200 } )
url = "http://localhost:8081/webgoat/attack"
req = gen_req(url,url)
f = opener.open(req, params)
write_f_to_f("f2.html",f)
f.close()

ourl = url
url = "http://localhost:8081/webgoat/attack?%s"%qs 
req = gen_req(url,ourl)
f = opener.open(req)
write_f_to_f("f3.html",f)
f.close()


ourl = url
url = "http://localhost:8081/webgoat/attack?%s"%qs
req = gen_req(url,ourl)
f = opener.open(req)
write_f_to_f("f4.html",f)
f.close()

NB: I did the answer to the assignment "the hard way" and handed it in. Now I'm just curious what kind of mechanism is stopping the urllib script from accessing the content

I tried accessing my local tomcat with webgoat and the webgoat console says:

Sat Feb 13 12:31:14 CET 2010 | 127.0.0.1:127.0.0.1 | org.owasp.webgoat.session.ErrorScreen | [Screen=107,menu=1200]
errorscreen createContent Error:null message:Invalid screen requested.  Try: http://localhost/WebGoat/attack
- WebGoat: Sat Feb 13 12:31:14 CET 2010 | 127.0.0.1:127.0.0.1 | org.owasp.webgoat.session.ErrorScreen | [Screen=107,menu=1200]
Sat Feb 13 12:31:14 CET 2010 | 127.0.0.1:127.0.0.1 | org.owasp.webgoat.session.ErrorScreen | [Screen=107,menu=1200]

It doesnt really help me as far as I can tell.

Answer 1

Is the user agent being checked? I've had to specify a user agent for a typical browser before to get past checks like that. You have one listed in your "data" variable but it looks like you don't actually apply it in any way.

Answer 2

Woohoo! Found out

The ID of the page I'm trying to find is regenerated on every session. So i need to look at the data returned before choosing a screenId:

import urllib2 
import urllib
import cookielib
import re

def write_f_to_f(fname,f):
    f1 = open(fname,"w")
    f1.write(f.read())
    f1.close()

def gen_req(url,referer=None):
    req = urllib2.Request(url)
    return req



myjar = cookielib.FileCookieJar("cookies.txt");
cookieHandler = urllib2.HTTPCookieProcessor(myjar)

password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
tlurl="http://127.0.0.1:8081/webgoat/attack"

password_mgr.add_password(None,tlurl,user="guest",passwd="guest")
find = "Blind Numeric."

authhandler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(authhandler,cookieHandler)

url = "http://127.0.0.1:8081/webgoat/attack"
req = gen_req(url,url)
f = opener.open(req)# Setup session and login.

params = urllib.urlencode({'start':'Start WebGoat'})
url = "http://127.0.0.1:8081/webgoat/attack"
req = gen_req(url,url)
f = opener.open(req,params) # Submit the "Start" form
dat = f.read() # Get the menu html from the firstpage.
f.close()


m = re.search("attack\?Screen=(\d+).*%s"%find,dat) # Try to find screen id for what i want
if not m is None:
    scr= m.group(1)
    qs = urllib.urlencode( {'Screen':scr, 'menu':1200 } )
    ourl = url
    url = "http://127.0.0.1:8081/webgoat/attack?%s"%qs
    req = gen_req(url,ourl)
    f = opener.open(req)
    write_f_to_f("f4.html",f)
    f.close()
    print "Found the page and saved it to f4.html"
else:
    print "Didnt find screen id for %s"%find