Hey so I have a small web scraper in my backend using express.js
const scrapeMetatags = (text) => {
const urls = Array.from( getUrls(text) );
const requests = urls.map(async url => {
const res = await fetch(url);
const html = await res.text();
const $ = cheerio.load(html);
const getMetatag = (name) =>
$(`meta[name=${name}]`).attr('content') ||
$(`meta[name="og:${name}"]`).attr('content') ||
$(`meta[name="twitter:${name}"]`).attr('content');
return {
url,
title: $('title').first().text(),
favicon: $('link[rel="shortcut icon"]').attr('href'),
// description: $('meta[name=description]').attr('content'),
description: getMetatag('description'),
image: getMetatag('image'),
author: getMetatag('author'),
}
});
return Promise.all(requests);
}
app.get("/scraper/:link", async (req, res) => {
const {link} = req.params;
const body = link
const data = await scrapeMetatags(body)
res.json(data)
})
Then in my frontend/react I'm calling it like so:
const [websitedescription, setwebsitedescription] = useState(null)
useEffect(() => {
async function scraper() {
const response = await fetch(`http://localhost:5000/scraper/${scraperlink}`)
const data = await response.json()
console.log(JSON.stringify(data))
const websitedescription = data[0].description
console.log(websitedescription)
const logolink = data[0].favicon
setwebsitedescription(websitedescription)
}
scraper()
}, [scraperlink])
The issue that I'm running into is that I'm getting a
Uncaught (in promise) TypeError: Cannot read properties of undefined (reading 'description') at scraper
right before the data gets populated. Which I believe might be because my component renders before the API has time to populate the data. I can confirm that the data actually gets called by looking in the console. I've put the returned object in stringified form so you can that what mean.
As you can see the error occurs right before the data is returned/populated. What can I do to fix this? thnks in advance