Im trying to scrap a web page, and after following some tutorials, i found how to scrap different products, and how to change the page, but not at the same time. I tryed some ways to do it, but couln't find out.
This is my scraping code:
const puppeteer = require('puppeteer');
const xlsx = require("xlsx");
async function getPageData(url,page){
await page.goto(url);
const h1 = await page.$eval(".product_main h1", h1 => h1.textContent);
const price = await page.$eval(".price_color", price => price.textContent);
const instock = await page.$eval(".instock.availability", instock => instock.innerText);
return {
title: h1,
price: price,
instock: instock
}
//await browser.close();
};
async function getLinks(){
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://books.toscrape.com/');
const links = await page.$$eval('.product_pod .image_container a', allAs =>
allAs.map(a => a.href));
}
async function main(){
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
const data = await getPageData("https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html",page);
console.log(data);
}
main();
And here is my code to change the page:
const puppeteer = require('puppeteer');
const xlsx = require("xlsx");
(async () => {
const browser = await puppeteer.launch({ headless: false });
const page = await browser.newPage();
await page.goto('https://books.toscrape.com/');
while(await page.$(".pager .next a")){
await page.click(".pager .next a");
await page.waitForTimeout(3000);
}
})();