I am trying to scrap a web page and give as response for my chatbot but I am getting empty response from dialogflow but when I log my scrapping object I get them. I am using glitch for my node js server.
How can I resolve this problem.
app.post("/webhook",async function(request,response){
var intentName = request.body.queryResult.intent.displayName;
if(intentName ="give_project"){
var type_project = request.body.queryResult.parameters['type_project'];
var project = await get_project(type_project);
//{url:.....,title:arduino,description: this is a arduino project}
console.log(project);
try{
response.json({
"fulfillmentText":`type project = `
})
}
})
async function get_project(type) {
const browser = await puppeteer.launch({
args: ['--no-sandbox', '--disable-setuid-sandbox'],
});
const page = await browser.newPage();
await page.goto('https://create.arduino.cc/projecthub/search?q=' + type);
const projects = await page.evaluate(() => {
let projectsFromWeb = [];
let elements = document.querySelectorAll('div.mobile-scroll-row-item');
for (let element of elements) {
if (element.querySelector('img') != null) {
projectsFromWeb.push({
url: element.querySelector('a.project-link-with-ref').href,
title: element.querySelector('span').textContent,
description: element.querySelector('p.description').innerText,
image: element.querySelector('img').src
}
);
} else {
projectsFromWeb.push({
url: element.querySelector('a.project-link-with-ref').href,
title: element.querySelector('span').textContent,
description: element.querySelector('p.description').innerText
});
}
}
return projectsFromWeb;
});
await browser.close();
return projects[getRandomNumber(projects.length)];
}
function getRandomNumber(max) {
return Math.floor(Math.random() * max);
};
// listen for requests
const listener = app.listen(process.env.PORT, () => {
console.log("Your app is listening on port " + listener.address().port);
});