im working on a webscraper for an api, and i was trying to divide it into functions, for whatever reason the await is not awaiting for the promise to be fulfilled
const PORT = 8000
const axios = require('axios')
const cheerio = require('cheerio')
const express = require('express')
const app = express()
const cors = require('cors')
app.use(cors())
const clarinURL = 'https://www.clarin.com/economia/'
app.get('/', function (req, res) {
res.json('This is my webscraper')
})
app.get('/results', async (req, res) => {
let results = await cheerioPick(clarinURL, '.content-nota', 'h2', 'a', 'https://www.clarin.com')
console.log(results)
res.send(results)
})
async function cheerioPick(url, container, titletag, urltag, domain) {
axios(url)
.then(response => {
const html = response.data
const $ = cheerio.load(html)
const articles = []
$(container, html).each(function () { //<-- cannot be a function expression
const title = $(this).find(titletag).text()
const url = domain + $(this).find(urltag).attr('href')
articles.push({
title,
url
})
})
console.log("articles")
return articles
})
.catch(err => console.log(err))
}
app.listen(PORT, () => console.log(`server running on PORT ${PORT}`))
on the console it prints:
undefined
articles
so it's clearly not awaiting for the async function to return it's result
The issue is that the function cheerioPick is not returning a Promise.
There are some ways to get this done. I would recomend use async/await in your axios segment so your whole code use the same approch of the asynchronous coding.
async function cheerioPick(url, container, titletag, urltag, domain) {
try {
const response = await axios(url);
const html = response.data
const $ = cheerio.load(html)
const articles = []
$(container, html).each(function () { //<-- cannot be a function expression
const title = $(this).find(titletag).text()
const url = domain + $(this).find(urltag).attr('href')
articles.push({
title,
url
})
})
console.log("articles")
return articles
} catch (err) {
console.log(err)
}