i am discovering puppeteer. I want to scrap some data with it, but this url return null value on querySelector. I tried to screenshot the page, error : you are not authorised to access this content. I tried to set the specifyAgent, but it's doesn't work. I tried some solutions on stackoverflow, but none of them worked. here's my code.
const puppeteer = require("puppeteer");
const url =
"https://www.cdiscount.com/juniors/lego/lego-star-wars-75192-millennium-falcon-ultimate/f-1202809-leg5702015869935.html?idOffre=1218498437#mpos=0|mp";
async function start() {
const browser = await puppeteer.launch();
//test the userAgent
//console.log(await browser.userAgent());
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/100.0.4889.0 Safari/537.36');
await page.goto(url);
await page.screenshot({path: 'headless-test-result.png'});
let data = await page.evaluate(() => {
return document.querySelector("span[itemprop=price]").innerText;
});
console.log("the price is " + data);
await browser.close();
}start()Here is the code with an example link that does not work.