I am trying to scrape the sports news from The Guardian UK using Cheerio. But, all I am getting is an empty string from the 'a' tag and 'undefined' for 'href' attribute.
const axios = require("axios")
const cheerio = require("cheerio")
const url = "https://www.theguardian.com/uk/sport"
const news = []
function getSportsNews() {
axios.get(url).then((response) => {
const html = response.data
const $ = cheerio.load(html)
$('a', html).each(() => {
const title = $(this).text()
const link = $(this).attr('href')
news.push({
title,
link
})
console.log(news)
})
})
}
getSportsNews()
I have tried adding a fake user-agent to the axios header, did not work.
However, it printed the texts when I ran console.log($('a').text()) outside the loop to check whether it was actually retrieving anything whatsoever. But in that each block it is returning all empty.