const puppeteer = require('puppeteer')
(async() => {
let url = 'https://www.reg.uci.edu/perl/WebSoc'
let browser = await puppeteer.launch({headless:false});
let page = await browser.newPage();
await page.goto(url, {waitUntil: 'domcontentloaded'});
await page.select('select[name=Dept]', 'BIO SCI');
await page.click('input[value="Display Web Results"]')
await page.waitForNavigation({waitUntil: 'domcontentloaded'})
await page.addScriptTag({path: 'jquery.js'})
var biologyClasses = await page.evaluate( () => {
var classes = $( "td:contains('Lec')")
classes = classes.not('.Comments')
classes = classes.not('.Pct100')
classes = Array.from(classes);
console.log(classes) // testing to see if classes show up
var biology = [];
classes.forEach(function(item) {
var classCode = window[$(item).closest("tr").find(">:nth-child(1)").text()]
classCode = {};
classCode.instructor = $(item).closest("tr").find(">:nth-child(5)").text()
biology.push(classCode)
});
return biology;
});
console.log(biologyClasses)
})();
I'm trying to scrape a list of biology classes and return an array of all the classes, with each class being an object that has the instructor of each class. Using headless:false, I see that it's navigating correctly but for some reason the code doesn't seem to be working inside of page.evaluate() and it's not returning anything.
When I manually copy the exact same page.evaluate() code to a regular, non-Puppeteer Chrome browser, the code seems to work and I get a proper array of 60 biology classes with each class being an object with an 'instructor' property. Any ideas what's going wrong when trying to emulate these results with Puppeteer?
Weirdly enough, the console.log(classes) statement inside of page.evauluate() randomly works at times (and I see classes logged within the Puppeteer browser) but even then, I don't get biologyClasses to be logged to the node.js console. Any ideas?