Ran into a problem that querySelector() returns null for an element that exists on the page.
I looked up stackover flow for some answers, I tried:
waitForSelector() & page.setUserAgent(UserAgent)
When I manually run the script document.querySelector('span.s-item__price') in the chrome console, it returned what I was expecting, but not when I tried with puppeteer.
const puppeteer = require('puppeteer');
async function openBrowser(url, callback) {
const browser = await puppeteer.launch({
headless: true,
defaultViewport: {
width: 1920,
height: 1080
}
});
const page = await browser.newPage();
await page.goto(`${url}`);
await callback(page);
await browser.close();
}
openBrowser('https://www.ebay.com/sch/i.html?_from=R40&_trksid=p2380057.m570.l1313&_nkw=men+watch&_sacat=0', async (page) => {
const products = await page.evaluate(() => {
let resultArray = [];
const productsArray = [...document.querySelectorAll('li.s-item.s-item__pl-on-bottom')];
for (let i = 1; i < productsArray.length; i++) {
resultArray.push({
id: i,
title: productsArray[i].querySelector('h3.s-item__title').textContent,
img: productsArray[i].querySelector('img.s-item__image-img').src,
price: productsArray[i].querySelector('span.s-item__price').textContent,
// seller: productsArray[i].querySelector('.s-item__seller-info-text').textContent THIS ONE FAILS, TOP ONES WORK FINE.
});
}
return resultArray;
});
console.log(products);
});