I m trying to scrape data from a website having multiple pages, the code works fine for a single run but when i try to put it in a loop, the website crashes and chrome says I m out of memory. Below is the javascript I m using :
// an XMLHttpRequest
var xhr = null;
var page = 1;
function scrape()
{
// instantiate XMLHttpRequest object
try
{
xhr = new XMLHttpRequest();
}
catch (e)
{
xhr = new ActiveXObject("Microsoft.XMLHTTP");
}
// handle old browsers
if (xhr == null)
{
alert("Ajax not supported by your browser!");
return;
}
// get city
var city = document.getElementById("city").value;
// get data
xhr.onreadystatechange =
function()
{
// only handle loaded requests
if (xhr.readyState == 4)
{
if (xhr.status == 200)
{
// insert link into DOM
some stuff
}
else
alert("Error with Ajax call!");
}
}
while(xhr.responseText != ""done)
{
// construct URL
var url = "scrape_data.php?city=" + city + "+" + page ;
xhr.open("GET", url, true);
xhr.send(null);
page++;
}
}
and this is the corresponding backend:
// get the q parameter
preg_match('/.*(?=\+)/',$_GET["city"],$city);
preg_match('/(?<=\+).*/',$_GET["city"],$page);
// scrape data from shiksha.com
$string = @file_get_contents("http://www.shiksha.com/b-tech/colleges/b-tech-colleges-".urlencode($city)."-{$page}");
if($page === 1)
{
// counting total number of pages
preg_match_all('/class=" linkpagination">/',$string,$result);
$GLOBALS["pages"] = sizeof($result[0]);
}
else $GLOBALS["pages"]--;
// passing the string for scraping data and storing in database
if($GLOBALS["pages"]=== 0)
print("loop");
now how do i loop through all the pages and also how do I check if all pages have been parsed (condition inside for loop). Thanks in advance for any help.