I am new to Scraping. This is my PDF downloading code. I want to use Async Await in this code. I don't know where I have to use async await in my code.
function scrapPdf(config, search_url, message) {
console.log('PDF downloading');
got(search_url).then(response => {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}).catch(err => {
console.log(err);
});
}
Here's one possibility assuming you only want to use async/await on the Promise returned from the .then:
async function scrapPdf(config, search_url, message) {
let response = null
console.log('PDF downloading');
try {
response = await got(search_url);
} catch (err) {
console.log(err);
}
if (response) {
const $ = cheerio.load(response.body);
$('.search-result').find('li > a').each((idx, elem) => {
if($(elem).text().trim() == 'PDF'){
const item = $(elem).attr('href');
pdf_lists.push(item);
}
})
$('ul.pagination').find('li.page-item').each((idx, elem) => {
if($(elem).attr('class').includes('page-item active navigation')){
if($(elem).next().hasClass('page-item navigation')){
scrapPdf(config, $(elem).next('li').find('a').attr('href'), message);
} else {
const search_result_dir = `./${message.date_ini}-${message.date_end}`;
if(!fs.existsSync(search_result_dir)){
fs.mkdirSync(search_result_dir)
}
for(let i = 0;i < pdf_lists.length; i++){
const download = new DownloaderHelper(pdf_lists[i], search_result_dir);
download.on('end', () => console.log('Download Completed'))
download.start();
}
console.log(`${pdf_lists.length} files Downloaded!`);
uploadFile(search_result_dir);
return ;
}
console.log($(elem).next('li').find('a').attr('href'));
}
});
}
}