I am an aspiring developer. As one of my projects, I am learning how to do web scraping. The goal is here to scrape a given webpage for any links that are PDFs and saving those links to a text file in NodeJS. With the given code, I am successfully console logging all of the matching links, but I am only getting one file written to my text file. Can someone steer me into the right direction?
const puppeteer = require("puppeteer");
const fs = require("fs/promises");
let myNewURL =
"https://www.renault.co.il/cars/Zoe/index.html?fbclid=IwAR1RtxbC_U2fImp9_KXJuQ869h5Wv77fyZVj8uBOU86rU90wb2L_NfrNppc";
async function scrapeSite(url) {
console.log("firing");
const browser = await puppeteer.launch({ headless: true });
const page = await browser.newPage();
await page.goto(url);
//this gives back an actual array, not a node list
const linkCollection = await page.$$eval("a", (links) => {
return links.map((link) => {
return link.href;
});
});
for (const link of linkCollection) {
if (link.includes(".pdf")) {
console.log(link);
await fs.writeFile("pdfLinks.txt", link);
}
}
await browser.close();
}
scrapeSite(myNewURL);
fs.writeFile overwrites the original file with each call. Try fs.appendFile instead. I've also added a newline (\n) at the end so the links are on individual lines:
for (const link of linkCollection) {
if (link.includes(".pdf")) {
console.log(link);
await fs.appendFile("pdfLinks.txt", link + '\n');
}
}
Alternatively, you could collect the links into an array first, then write them all together:
const pdfLinks = [];
for (const link of linkCollection) {
if (link.includes(".pdf")) {
console.log(link);
pdfLinks.push(link);
}
}
const output = pdfLinks.join('\n')
await fs.writeFile("pdfLinks.txt", output);
use append flag:
await fs.writeFile("pdfLinks.txt", link+'\n',{flag:'a'});