I have a code problem that I could not fix for 2 days, here is my code:
const crawl = async () => {
try {
const link = 'https://www.olx.pl/nieruchomosci/mieszkania/wynajem/wroclaw/?search%5Bfilter_float_price%3Ato%5D=3000&search%5Bfilter_float_price%3Afrom%5D=1000&search%5Bfilter_enum_rooms%5D%5B0%5D=two';
const browser = await puppeteer.launch({
'args': [
'--no-sandbox',
'--disable-setuid-sandbox'
]
});
const [page] = await browser.pages();
await page.goto(link);
// irrelevant stuff //
for (let href of correctAdverts) {
let hrefChecked = await checkForPromotedAdvert(href);
console.log('entering ' + href);
await page.goto(href, {
waitUntil: 'networkidle2',
timeout: 0
});
pool.query('SELECT id FROM data WHERE href = $1', [hrefChecked], (err, res) => {
if (err) {
console.log(err.stack)
} else if (res.rows[0]) {
console.log("already in DB, " + JSON.stringify(res.rows[0]))
} else {
insertData(page, href)
}
})
}
await browser.close();
} catch (err) {
console.error(err);
}
}
And this is my insertData function:
const insertData = async (page, href) => {
const olxPriceSelector = process.env.OLX_PRICE_SELECTOR;
const olxAdditionalPaymentsSelector = process.env.OLX_ADDITIONAL_PAYMENTS_SELECTOR;
const olxAreaSelector = process.env.OLX_PRICE_SELECTOR;
const olxDistrictSelector = process.env.OLX_DISTRICT_SELECTOR;
if (href.startsWith('https://www.olx.pl/d/oferta/')) {
let price = await getData(page, olxPriceSelector, null);
let additionalPayments = await getData(page, olxAdditionalPaymentsSelector, 'Czynsz');
let area = await getData(page, olxAreaSelector, 'Powierzchnia: ');
let district = await getData(page, olxDistrictSelector, null, true);
if (!district) {
if (await page.$(olxDistrictSelector) !== null) {
district = await page.evaluate(el => el.textContent, await page.$(olxDistrictSelector));
} else district = 'undefined';
}
pool.query("INSERT INTO data (price,href,additional_payments,area,district) VALUES ($1,$2,$3,$4,$5) ON CONFLICT (href) DO NOTHING;", [price, href, additionalPayments, area, district], function () {})
}
}
And when I run it I get error : Execution context was destroyed, most likely because of a navigation.
Firstly I thought the page.close after the for of loop was the problem, but commenting it out doesn't seem to be fixing the problem
Can you see where might be the error?