I'm trying to run approximately 1,000 queries on MongoDB to check for matches on particular object property.
Please shield your eyes from what is no doubt some extremely amateur code but would appreciate any ways to make this more efficient. It works but takes a long time.
'followers' is an array of Twitter user objects and it is being checked against a database of half a million records.
function checkFollowers(followers){
// check if followers are in DB
let matches = [];
MongoClient.connect(url, function(err, db) {
if (err) throw err;
console.log("Connected!");
async.each(followers, function(item, callback){
var regex = new RegExp("^" + item.username + "$", 'i')
dbo.collection("User").findOne({Username : {$regex : regex}},function(err, result) {
if (err) throw err;
if(result != null){
matches.push(result.Username)
}
,
callback(null);
});
}, function(err){
if(err) {
console.log(err);
} else {
console.log('All records have been checked');
db.close();
console.log("This many matching records " + matches.length)
console.log(matches)
}
});
});
}
Maybe like this:
const allUsers = dbo.collection("User").find().toArray();
followers.forEach( follower => {
var regex = new RegExp("^" + follower.username + "$", 'i')
if (allUsers.filter( aUser = > regex.test(aUser)))
matches.push(aUser.Username);
})
Reading the collection again and again does not make much sense.
Regular expressions are always slow, so you may prefer
dbo.collection("User").aggregate([
{$set: {username_UC: { $toUpper: "$username" } } }
]).toArray();
followers.forEach( follower => {
if (allUsers.filter( aUser => aUser.username_UC == follower.toUpperCase() ))
matches.push(aUser.Username);
})
Or even faster:
dbo.collection("User").aggregate([
{$match: {$expr: { $in: [ { $toUpper: "$username" }, followers.map(x => x.toUpperCase()) ] } } }
])