I'm using puppeteer to scrape a job website. Everything is working so far.
app.get('/', async (req, res) => {
res.render('index', {text: 'This is the index page'});
})
On submit the user gets redirected to an ejs file
<body>
<div class="container">
<input type="text" class="submit" id="city-input" placeholder="Submit City">
<button type="submit" onclick="submitCity()">Submit</button>
</div>
</body>
script.js
function submitCity() {
const city = document.getElementById('city-input').value;
const url = 'http://localhost:3000/api/germany/' + city
window.location.href = url;
console.log(city);
}
index.js
app.get('/api/germany/:key', async (req, res) => {
const city = req.params.key;
const data = await scraper.scrapeData(city);
await db.updateDB(data);
await res.render('result', { data: data });
})
So when the user clicks on submit it can take about 1-2 minutes until the result page is ready. I want to add a loading page/information until the data is scraped and ready to send.
You can use fetch to make the request in the background:
function submitCity() {
const city = document.getElementById('city-input').value;
const url = 'http://localhost:3000/api/germany/' + city
const response = await fetch(url); // make initial request
const data = await response.json(); // wait for response body, then parse as json
console.log(data);
}
And on the server, just return the data to the request:
app.get('/api/germany/:key', async (req, res) => {
const city = req.params.key;
const data = await scraper.scrapeData(city);
await db.updateDB(data);
return res.json(data);
});