Estoy extrayendo datos de lugares de Google Maps, pero lo que sucede es que solo me devuelve los primeros 10 resultados de las reseñas de los usuarios, no después de eso. Creo que hay algún problema con la funcionalidad de desplazamiento.
const puppeteer = require('puppeteer'); function extractItems() { const extractedElements = document.querySelectorAll('.MyEned span.wiI7pd'); const items = []; for (let element of extractedElements) { items.push(element.innerText); } return items; } async function scrapeItems( page, extractItems, itemCount, scrollDelay = 2000, ) { let items = []; try { let previousHeight; while (items.length < itemCount) { items = await page.evaluate(extractItems); previousHeight = await page.evaluate('div.m6QErb.DxyBCb.scrollHeight');//selector for scroller await page.evaluate('window.scrollTo(0, div.m6QErb.DxyBCb.scrollHeight)'); await page.waitForFunction(`div.m6QErb.DxyBCb.scrollHeight > ${previousHeight}`); await page.waitForTimeout(scrollDelay); } } catch(e) { } return items; } (async () => { let browser = await puppeteer.connect(); browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const [page] = await browser.pages(); page.setViewport({ width: 1280, height: 926 }); await page.goto('https://www.google.com/maps/place/Ace+Florist+%26+Flower+Delivery/@40.8265438,-73.5011026,15z/data=!4m7!3m6!1s0x0:0x9062074cae10c10f!8m2!3d40.8265438!4d-73.5011026!9m1!1b1'); // Auto-scroll and extract desired items from the page. Currently set to extract eight items. const items = await scrapeItems(page, extractItems, 30); console.log(items) await browser.close(); })();Este código está funcionando bien:
'use strict' const puppeteer = require('puppeteer'); function extractItems() { const extractedElements = document.querySelectorAll('.MyEned span.wiI7pd'); const items = []; for (let element of extractedElements) { items.push(element.innerText); } return items; } async function scrapeItems( page, extractItems, itemCount, scrollDelay = 2000, ) { let items = []; try { let previousHeight; while (items.length < itemCount) { console.log(`items.length: ${items.length} itemCount: ${itemCount}`) items = await page.evaluate(extractItems); previousHeight = await page.evaluate(() => { const scroller = document.querySelector('div.m6QErb.DxyBCb') return scroller.scrollHeight }) await page.evaluate(`document.querySelector("div.m6QErb.DxyBCb").scrollTo(0, ${previousHeight})`); await page.waitForFunction(`document.querySelector("div.m6QErb.DxyBCb").scrollHeight > ${previousHeight}`); await page.waitForTimeout(scrollDelay); } } catch(e) { } return items; } (async () => { const browser = await puppeteer.launch({ headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const [page] = await browser.pages(); page.setViewport({ width: 1280, height: 926 }); await page.goto('https://www.google.com/maps/place/Ace+Florist+%26+Flower+Delivery/@40.8265438,-73.5011026,15z/data=!4m7!3m6!1s0x0:0x9062074cae10c10f!8m2!3d40.8265438!4d-73.5011026!9m1!1b1'); // Auto-scroll and extract desired items from the page. Currently set to extract eight items. const items = await scrapeItems(page, extractItems, 30); console.log(items) await browser.close(); })();Así que descubrí que tengo que agregar document.querySelector mientras evalúo la altura de desplazamiento y también cuando verifico que la altura de desplazamiento es mayor que la altura anterior.
items = await page.evaluate(extractItems); previousHeight = page.evaluate('document.querySelector("div.m6QErb.DxyBCb").scrollHeight'); await page.evaluate(`document.querySelector("div.m6QErb.DxyBCb").scrollTo(0, ${previousHeight[0]})`); await page.waitForFunction(`document.querySelector("div.m6QErb.DxyBCb").scrollHeight > ${previousHeight[0]}`); await page.waitForTimeout(scrollDelay);