I'm trying to scrape the same data off multiple pages of a website. The url links are all in a json document as an array. I can't figure out how to loop through each website link and grab data from each page. I feel like I need to make a loop for the entire function but I want to make sure the data all ends up in the same file. Here's what I have thus far...
const fs = require('fs');
const puppeteer = require('puppeteer');
function extractItems() {
const extractedElements = document.querySelectorAll('#MoreInfoPanel_81 > div.more-info-panel-body');
const items = [];
for (let element of extractedElements) {
items.push(element.innerText);
}
return items;
}
async function scrapeItems(
page,
extractItems,
itemCount,
scrollDelay = 800,
) {
let items = [];
try {
let previousHeight;
while (items.length < itemCount) {
items = await page.evaluate(extractItems);
previousHeight = await page.evaluate('document.body.scrollHeight');
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
await page.waitForFunction(`document.body.scrollHeight > ${previousHeight}`);
await page.waitForTimeout(scrollDelay);
}
} catch(e) { }
return items;
}
function hospitalLinks() {
let dataFile = require('./hospitallinks.json');
for (let i = 0; i < dataFile.length; i++) {
}
}
(async () => {
// Set up Chromium browser and page.
const browser = await puppeteer.launch({
headless: false,
args: ['--no-sandbox', '--disable-setuid-sandbox'],
});
const page = await browser.newPage();
page.setViewport({ width: 1280, height: 926 });
// Navigate to the page.
await page.goto(hospitalLinks());
// Auto-scroll and extract desired items from the page.
const items = await scrapeItems(page, extractItems, 4908);
// Save extracted items to a new file.
fs.writeFileSync('./facilitydata.txt', items.join('\n') + '\n');
// Close the browser.
await browser.close();
})();
It works if I replace the hospitalLinks() with a url...I tried using forEach but it didn't work either.