I am currently trying to speed up my server, which uses puppeteer to scrape information whenever it receives requests from a chrome extension I am building. However, it is still quite slow despite me trying to run it concurrently. Is there a way to keep my puppeteer browser open once my server starts, so I can shave off the time it takes to open the browser each time a request is sent?
const withBrowser = async (fn) => {
const browser = await puppeteer.launch({args: ['--no-sandbox']});
try {
return await fn(browser); }
finally {
await browser.close();
}
}
const withPage = (browser) => async (fn) => {
const page = await browser.newPage();
// await page.setDefaultNavigationTimeout(0);
// await page.setCacheEnabled(false);
try {
return await fn(page);
} finally {
await page.close();
}
}
async function processing(urls){
return await withBrowser(async (browser) => {
return bluebird.map(urls, async (url) => {
return withPage(browser)(async (page) => {
await page.goto(url);
let texts = await page.evaluate(() => {
let data = [];
// do work
}
return ["0000"];
}
);
}, {concurrency: 4});
});
}