I have a large array of addresses that I need to open at the same time and get data. due to server limitation, I cannot open more than 10 tabs at the same time. How to wait until a new free page appears in the serialNumbers.forEach? I understand that I can check if browser.pages () < 10. But how to make the loop wait in one iteration of a free tab and not go to the next iteration?
const puppeteer = require('puppeteer');
let numberOfOpenPages;
let serialNumbers = ['FVFDT2GCQ6L7', 'DLXFG3X2DFJ1', 'FVFDT2GCQ6L7', 'DLXFG3X2DFJ1', 'FVFDT2GCQ6L7', 'DLXFG3X2DFJ1'];
async function processArray() {
const browser = await puppeteer.launch({ headless: true });
serialNumbers.forEach(async (item, i) => {
// new page and then get data
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36');
await page.goto(`https://support.apple.com/en_US/specs/${item}`);
await page.waitForSelector('.article_link');
await page.click('.article_link');
await page.waitForSelector('#article');
const specs = await page.evaluate(() => {
let article = document.querySelector('h1').innerHTML;
return `${article}`;
});
console.log(specs);
await page.close();
// close browser when all pages is closed
numberOfOpenPages = (await browser.pages()).length;
console.log(numberOfOpenPages);
if (numberOfOpenPages <= 1) {
await browser.close();
}
})
}
processArray()
Maybe something like this?
const browser = await puppeteer.launch({ headless: true });
const serialNumbers = ['FVFDT2GCQ6L7', 'DLXFG3X2DFJ1', 'FVFDT2GCQ6L7', 'DLXFG3X2DFJ1', 'FVFDT2GCQ6L7', 'DLXFG3X2DFJ1'];
const pageLimit = 10;
for (let i = 0; i < pageLimit; i++) {
processPage(await browser.newPage());
}
async function processPage(page) {
if (serialNumbers.length === 0) {
await page.close();
if ((await browser.pages()).length <= 1) await browser.close();
return;
}
const serialNumber = serialNumbers.shift();
await page.goto(`https://support.apple.com/en_US/specs/${serialNumber}`);
// Process the page...
processPage(page);
}
I made it using setTimeOut but sometimes it launches more tabs at the same time than it needs to. But the error is small.
If there are more correct solutions, I will be glad to know
// const { copyFileSync } = require('fs');
const puppeteer = require('puppeteer');
let numberOfOpenPages;
let i = 0;
let serialNumbers = ['FVFDT2GCQ6L7', 'DLXFG3X2DFJ1', 'FVFDT2GCQ6L7', 'DLXFG3X2DFJ1', 'FVFDT2GCQ6L7', 'DLXFG3X2DFJ1', 'FVFDT2GCQ6L7', 'DLXFG3X2DFJ1', 'FVFDT2GCQ6L7'];
async function newPage(serial, browser) {
numberOfOpenPages = (await browser.pages()).length;
console.log(numberOfOpenPages);
if (numberOfOpenPages < 10) {
// new page and then get data
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36');
await page.goto(`https://support.apple.com/en_US/specs/${serialNumbers[serial]}`);
await page.waitForSelector('.article_link');
await page.click('.article_link');
await page.waitForSelector('#article');
const specs = await page.evaluate(() => {
let article = document.querySelector('h1').innerHTML;
return `${article}`;
});
console.log(specs);
await page.close();
// close browser when all pages is closed
numberOfOpenPages = (await browser.pages()).length;
// console.log(numberOfOpenPages);
// if (numberOfOpenPages <= 1) {
// await browser.close();
// console.log('браузер закрыт!')
// }
} else {
setTimeout(() => {
newPage(serial, browser);
}, 3000); // не ставить меньше 1 сек чтобы не совпадало сразу в один свободный слот
}
}
async function processArray() {
const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'], headless: true });
// set your counter to 1
function myLoop () { // create a loop function
setTimeout(function () { // call a 3s setTimeout when the loop is called
console.log('запускаем', i);
newPage(i, browser); // your code here
i++; // increment the counter
if (i < serialNumbers.length) { //if the counter < 10, call the loop function
myLoop(); // .. again which will trigger another
} else console.log('end loop'); // .. setTimeout()
}, 500)
}
myLoop();
}
processArray()