My code was working and the error: Error: Evaluation failed: TypeError: Cannot read properties of null (reading 'innerText') came back. I don't understand why. I try to get the text contained in the tags and send it in a JSON document Here is my code:
const scraperObject = {
url: 'https://stockx.com/fr-fr/dior-b713-cactus-jack-mocha',
async scraper(browser){
let page = await browser.newPage();
console.log(`Navigating to ${this.url}...`);
await page.goto(this.url);
await page.waitForSelector('#onetrust-reject-all-handler');
await page.click('#onetrust-reject-all-handler');
const result = await page.evaluate(() => {
let demandes = document.querySelector('#main-content > div > section:nth-child(3) > div.css-gg4vpm > div.css-0 > div.css-qt7qal > div.chakra-stack.css-1g42e87 > div > a.chakra-button.css-2zzyy5 > p').innerText;
let offres = document.querySelector('#main-content > div > section:nth-child(3) > div.css-gg4vpm > div.css-0 > div.css-qt7qal > div.chakra-stack.css-1g42e87 > a > p').innerText;
return {demandes, offres}
})
console.log(result);
browser.close()
return result
}
}
module.exports = scraperObject;
Here is what was displayed in my JSON file when the code was running:
{"demandes":"Acheter à 1 365 €","offres":"Vendre à 784 € ou demander plus"}
My JSON is controlled in this file "pageController" :
const pageScraper = require('./pageScraper');
const fs = require('fs');
async function scrapeAll(browserInstance){
let browser;
try{
browser = await browserInstance;
const scrapedData = await pageScraper.scraper(browser)
fs.writeFile("data.json", JSON.stringify(scrapedData), 'utf8', function(err) {
if(err) {
return console.log(err);
}
console.log("The data has been scraped and saved successfully! View it at './data.json'");
});
console.log(scrapedData)
}
catch(err){
console.log("Could not resolve the browser instance => ", err);
}
}
module.exports = (browserInstance) => scrapeAll(browserInstance)
And I don't know how to modify it
Thanks in advance
I'm not a big fan of those massive browser-generated selectors. If just one class or element in the chain changes, the whole thing breaks.
Easier than messing with the DOM is intercepting the response that has the data JSON in it:
const fs = require("fs").promises;
const puppeteer = require("puppeteer"); // ^15.4.0
let browser;
(async () => {
browser = await puppeteer.launch();
const [page] = await browser.pages();
const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36";
await page.setUserAgent(ua);
const url = "https://stockx.com/fr-fr/dior-b713-cactus-jack-mocha";
const responseP = page.waitForResponse(async response => {
if (response.url() === "https://stockx.com/api/p/e") {
const payload = await response.json();
return payload?.data?.product?.market?.bidAskData;
}
});
await page.goto(url, {waitUntil: "domcontentloaded"});
const result = await (await responseP).json();
await fs.writeFile("result.json", JSON.stringify(result, null, 2));
console.log(result.data.product.market.bidAskData);
})()
.catch(err => console.error(err))
.finally(() => browser?.close())
;
Output:
{
highestBid: 803, // vendre / offres
highestBidSize: '43',
lowestAsk: 1332, // acheter / demandes
lowestAskSize: '42',
__typename: 'BidAskData'
}
My numbers are bit different than yours since I'm in the USA.
This doesn't have the fancy text, but that probably saves a step for you.