cURL + JS_Extractor + php + MYSQL mess = Please help

Question

A little background:
There is a game I play on the net called doomlord...
I'm in a Clan. I'm trying to present stats on the players that are relevant for us.
For this I have to retrieve the names of the players from a web page that I can only access after a login. The page is:
www.doomlord.net/index.php?m=szovetseg&sub=reszletes_statisztikak
but this will take you only to the login page of the game and not to the real page.

To get to the real page I use cURL (to login and get a cookie that lets me roam the pages). To retrieve the data from a table I use JS_extractor class.

The problem:
Separately they work, but when used together they crap out.

What did I try:

1)
I used cURL and shown that I was able to retrieve the page I need
see here: doomlord.tamtek.net/gettable.php
2)
I added js_extractor and cURL gets the page but js_extractor do not retrieve the data
see here: doomlord.tamtek.net/gettable2.php
3)
I logged in manually to the doomlord website, I went to the page I need, I viewed and then "copy&paste"-ed the source. Then I uploaded the source as an html page into my website where I used js_extractor and it perfectly got the table I need into an array for further manupulation...
The page source: doomlord.tamtek.net/doomtest2.html
The js_extractor result: doomlord.tamtek.net/gettable3.php

And for the code:

(please excuse some of the extra write-outs, but I'm trying to make it as visual as possible)

This is the cURL code I use:

$ckfile = tempnam ("tmp", "cookie.tmp");

$fields_string='';
$fields = array(
                  'username'=>urlencode(SITE_USER),
                  'jelszo'=>urlencode(SITE_PASS),
                  'vilag'=>urlencode(SITE_WORLD),
                  'tev'=>urlencode(SITE_TEV),
            );

foreach($fields as $key=>$value) { $fields_string .= $key.'='.$value.'&'; }
//rtrim($fields_string,'&');
$fields_string=substr($fields_string,0,-1);
//print_r($fields_string);
$ch=curl_init();
curl_setopt($ch,CURLOPT_USERAGENT,"Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1" );
curl_setopt($ch,CURLOPT_URL,'http://www.doomlord.net');
curl_setopt($ch,CURLOPT_COOKIEJAR,$ckfile);
curl_setopt($ch,CURLOPT_POST,count($fields));
curl_setopt($ch,CURLOPT_POSTFIELDS,$fields_string);
curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,20);
curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
curl_setopt($ch,CURLOPT_FOLLOWLOCATION,3);
curl_setopt($ch,CURLOPT_MAXREDIRS,10);
$connect=curl_exec($ch);
$response1=curl_getinfo( $ch );
curl_close($ch);

  $ch=curl_init();
  curl_setopt($ch,CURLOPT_USERAGENT,"Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1" );
  curl_setopt($ch,CURLOPT_URL,'http://www.doomlord.net/index.php?m=szovetseg&sub=reszletes_statisztikak');
  curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,20);
  curl_setopt($ch,CURLOPT_COOKIEFILE,$ckfile);
  curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
  curl_setopt($ch,CURLOPT_FOLLOWLOCATION,3);
  curl_setopt($ch,CURLOPT_MAXREDIRS,10);
  $connect=curl_exec($ch);
  $response1=curl_getinfo( $ch ); 
  print_r($connect);
//  print_r($response1);

curl_close($ch);

This is the cURL and js_extractor code together:

<?php
session_start();

require_once('inc/constant.php');
require_once('inc/function.php');  //basic functions

$ckfile = tempnam ("tmp", "cookie.tmp");

$fields_string='';
$fields = array(
                  'username'=>urlencode(SITE_USER),
                  'jelszo'=>urlencode(SITE_PASS),
                  'vilag'=>urlencode(SITE_WORLD),
                  'tev'=>urlencode(SITE_TEV),
            );

foreach($fields as $key=>$value) { $fields_string .= $key.'='.$value.'&'; }
//rtrim($fields_string,'&');  //tried both commented out and not 
$fields_string=substr($fields_string,0,-1);
$ch=curl_init();
curl_setopt($ch,CURLOPT_USERAGENT,"Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1" );
curl_setopt($ch,CURLOPT_URL,'http://www.doomlord.net');
curl_setopt($ch,CURLOPT_COOKIEJAR,$ckfile);
curl_setopt($ch,CURLOPT_POST,count($fields));
curl_setopt($ch,CURLOPT_POSTFIELDS,$fields_string);
curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,20);
curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
curl_setopt($ch,CURLOPT_FOLLOWLOCATION,3);
curl_setopt($ch,CURLOPT_MAXREDIRS,10);
$connect=curl_exec($ch);
$response1=curl_getinfo( $ch );
curl_close($ch);

  $ch=curl_init();
  curl_setopt($ch,CURLOPT_USERAGENT,"Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1" );
  curl_setopt($ch,CURLOPT_URL,'http://www.doomlord.net/index.php?m=szovetseg&amp;sub=reszletes_statisztikak');
  curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,20);
  curl_setopt($ch,CURLOPT_COOKIEFILE,$ckfile);
  curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
  curl_setopt($ch,CURLOPT_FOLLOWLOCATION,3);
  curl_setopt($ch,CURLOPT_MAXREDIRS,10);
  $connect=curl_exec($ch);
  $response1=curl_getinfo( $ch ); 
  print_r($connect); // just to show that i have connected
//  print_r($response1);

set_include_path(get_include_path() . PATH_SEPARATOR . './library/');
require_once 'JS/Extractor.php';

//$extractor = new JS_Extractor(file_get_contents('http://www.doomlord.net/index.php?m=szovetseg&amp;sub=reszletes_statisztikak')); 
$extractor = new JS_Extractor(file_get_contents($connect)); // tried both ways
echo "</br />";
echo "This is the extractor:</br />";
var_dump($extractor);
echo "</br />";
echo "</br />";
$body = $extractor->query("body")->item(0);
echo "</br />";
echo "This is the body:</br />";
var_dump($body);
echo "</br />";
echo "</br />";
$table = $body->query("//table[@class=\"rstatisztika_tabla\"]")->item(0);
echo "</br />";
echo "This is the table:</br />";
var_dump($table);
echo "</br />";
echo "</br />";
$data = $table->extract(array("tr", "td"));
echo "</br />";
echo "This is the data:</br />";
var_dump($data);
echo "</br />";
echo "</br />";

curl_close($ch);
//}
?>

This is the js_extactor code I use to get the table after I placed the source on my site:

<?php

set_include_path(get_include_path() . PATH_SEPARATOR . './library/');
require_once 'JS/Extractor.php';

$extractor = new JS_Extractor(file_get_contents('http://doomlord.tamtek.net/doomtest2.html'));
echo "</br />";
echo "This is the extractor:</br />";
var_dump($extractor);
echo "</br />";
echo "</br />";
$body = $extractor->query("body")->item(0);
echo "</br />";
echo "This is the body:</br />";
var_dump($body);
echo "</br />";
echo "</br />";
$table = $body->query("//table[@class=\"rstatisztika_tabla\"]")->item(0);
echo "</br />";
echo "This is the table:</br />";
var_dump($table);
echo "</br />";
echo "</br />";
$data = $table->extract(array("tr", "td"));
echo "</br />";
echo "This is the data:</br />";
var_dump($data);
echo "</br />";
echo "</br />";
?>

and the really weird part:
When I run the script like this:

$extractor = new JS_Extractor(file_get_contents($connect)); // this is line 53

, I also get this piece of beauty:

*Warning*: file_get_contents(<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://...@u="; var wa_referrer = "@r="; if(wa.WACID==null) { wa.WACID=wa.generateID('A'); wa.setCookie(wa.WACIDName,wa.WACID,wa.getTopDomain(wa.getDomain(document.URL))); } same = same + "@c=" + wa.WACID; if(screen) felbontas='@s='+screen.width+'x'+screen.height; if(document.referrer) wa_referrer=wa_referrer+document.referrer; if(document.URL) wa_url=wa_url+document.URL; same = same + felbontas + wa_url + wa_referrer; //--> </SCRIPT> </head> <body> <SCRIPT language="JavaScript"> <!-- document.write('<!-- Medián WebAudit HarmoNet Vegzetur 1/2 --><img style="position:absolute;top:-100px;left:-100px" src="http://audit.median.hu/cgi-bin/track.cgi?uc=12283086407878&amp;dc=1&amp;ui='+same+'" width="1" height="1">'); //--> </SCRIPT> <NOSCRIPT> <!-- M in */home2/tamtek/public_html/doomlord/gettable2.php* on line *53*

which is interesting because it drops like 57 lines of code from the webpage between

*Warning*: file_get_contents(<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://

and

@u="; var wa_referrer = "@r=";

on the first line... then prints some more, and then ignores the rest... and gives me an error on Line 53

I suppose, it runs into an interpretation of a special character that makes the rest of the page disappear, but I have no clue what it could be and how to get rid of it...

Does anyone have any suggestions on what could be wrong?

By the way this is where you can find JS Extractor... jacksleight.com/old/blog/2008/02/10/js-extractor-and-the-death-of-table-extractor

And if you like to play a FREE Game and really want to help, you can join here:
www.doomlord.net/?kar=147
You need only an email address and nothing else.
Make sure you choose the diamond horde (otherwise you will not be able to join my clan and make this a futile excercise)
And Join my clan which is "Girlfight" (this is the only way for you to get to the original page www.doomlord.net/index.php?m=szovetseg&sub=reszletes_statisztikak

But even without joining I think I give enough data here to hopefulle see what is goping on..

If you think you can help but you need more data, please feel free to ask...

Thank you.

Thadson

Answer 1

WOW, I just solved my own problem...
Who would have thought...

Instead of trying to get the table data directly, and fail miserably, I write the file out into a temporary file and then pull the table data from there...

Remember that I told you that the script worked on a local file?
So I made the file my local file :-)

Here is what I did:

<?php
session_start();

require_once('inc/constant.php');
require_once('inc/function.php');  //basic functions
db_on();

$ckfile = tempnam ("tmp", "cookie.tmp");

$fields_string='';
$fields = array(
                  'username'=>urlencode(SITE_USER),
                  'jelszo'=>urlencode(SITE_PASS),
                  'vilag'=>urlencode(SITE_WORLD),
                  'tev'=>urlencode(SITE_TEV),
            );

foreach($fields as $key=>$value) { $fields_string .= $key.'='.$value.'&'; }
//rtrim($fields_string,'&');
$fields_string=substr($fields_string,0,-1);
//print_r($fields_string);
$ch=curl_init();
curl_setopt($ch,CURLOPT_USERAGENT,"Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1" );
curl_setopt($ch,CURLOPT_URL,'http://www.doomlord.net');
curl_setopt($ch,CURLOPT_COOKIEJAR,$ckfile);
curl_setopt($ch,CURLOPT_POST,count($fields));
curl_setopt($ch,CURLOPT_POSTFIELDS,$fields_string);
curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,20);
curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
curl_setopt($ch,CURLOPT_FOLLOWLOCATION,3);
curl_setopt($ch,CURLOPT_MAXREDIRS,10);
$connect=curl_exec($ch);
$response1=curl_getinfo( $ch );
$fp=fopen('temp.html','w'); //create my temp file here
fclose($fp);
curl_close($ch);

  $ch=curl_init();
  curl_setopt($ch,CURLOPT_USERAGENT,"Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1" );
  curl_setopt($ch,CURLOPT_URL,'http://www.doomlord.net/index.php?m=szovetseg&amp;sub=reszletes_statisztikak');
  curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,20);
  curl_setopt($ch,CURLOPT_COOKIEFILE,$ckfile);
  curl_setopt($ch,CURLOPT_RETURNTRANSFER,true);
  curl_setopt($ch,CURLOPT_FOLLOWLOCATION,3);
  curl_setopt($ch,CURLOPT_MAXREDIRS,10);
  $connect=curl_exec($ch);
  $response1=curl_getinfo( $ch ); 
// and this is the key here
  $fp=fopen('temp.html','a');
  fputs($fp, $connect);
  fclose($fp);
curl_close($ch);

// now my the extractor works
set_include_path(get_include_path() . PATH_SEPARATOR . './library/');
require_once 'JS/Extractor.php';

$extractor = new JS_Extractor(file_get_contents('temp.html'));
$body = $extractor->query("body")->item(0);
$table = $body->query("//table[@class=\"rstatisztika_tabla\"]")->item(0);
$data = $table->extract(array("tr", "td"));
echo "</br />";
echo "This is the data:</br />";
var_dump($data);
echo "</br />";
echo "</br />";
?>