I have inserted 100k records, I have uploaded records batch-wise successfully. But, If I use the bulk.insert(doc) the default value not inserted using mongoose and Nodejs. like createdAt and updatedAt field as default value not inserted. I trying to add the options setDefaultsOnInsert: true bulk.insert no option to add the value. Currently, I added my code. Please help me out for advance.
let data = req.body;
var bulk = callDispositionModel.collection.initializeOrderedBulkOp();
var counter=0
data.forEach(doc1 => {
bulk.insert(doc1);
if (counter % 5000 == 0) {
bulk.execute();
bulk = callDispositionModel.collection.initializeUnorderedBulkOp();
counter = 0;
}
counter++
})
if (counter > 0) {
bulk.execute(function(err,result) {
if(err){
console.log(`err `, err)
}else{
console.log(`result `, result)
return res.send({success: true, message:'data uploaded successfully')
}
});
}
let dispositionSchema = new mongoose.Schema({
name : {type: String, default: null},
mobile : {type : String, default: null},
remarks : {type: String, default:null},
duration: {type : String, default: null},
amount : {type : Number, default: 0},
date : {type : String, default: null},
time : {type : String, default: null},
createdAt: {type: Date, default: Date.now },
updatedAt: {type: Date, default: Date.now }
});
const disposition = mongoose.model('disposition', dispositionSchema);
export default disposition;
It's inserted the data in mongodb
{
"_id" : ObjectId("6098e6d007e2804b9c1f8317"),
"name" : "senthil",
"amount" : 0
}
{
"_id" : ObjectId("6098e6d007e2804b9c1f8316"),
"name" : "periyas",
"amount" : 0
}
But, I have expected the output
Expected Data
{
"_id" : ObjectId("6098e6d007e2804b9c1f8317"),
"name" : "senthil",
"amount" : 0,
"mobile" : null,
"remarks" : null,
"createdAt": "2021-05-07T13:55:34.233Z"
},
{
"_id" : ObjectId("6098e6d007e2804b9c1f8316"),
"name" : "periyas",
"mobile" : null,
"remarks" : null,
"createdAt": "2021-05-07T13:55:34.233Z"
}
There is a bulk insert support request in mongoose github, and read one of the comment by moderator:
The
initializeUnorderedBulkOp()andinitializeOrderedBulkOp()api methods, which is "soft deprecated" by mongodb. It has been replaced by the bulkWrite() API, which is part of MongoDB's core CRUD spec and thus is much more widely implemented. In particular, mongoose has a Model.bulkWrite() function as of 4.9.0 that has validation, casting, promises, and ref depopulating.
You can use bulkWrite() something like:
let data = req.body;
let bulk = [];
data.forEach((doc1) => {
bulk.push({ "insertOne": { "document": doc1 } });
if (bulk.length === 5000) {
callDispositionModel.bulkWrite(bulk).then((result) => {});
bulk = [];
}
})
if (bulk.length > 0) {
callDispositionModel.bulkWrite(bulk).then((res) => {
console.log(res.insertedCount);
return res.send({success: true, message: 'all data uploaded successfully'})
});
}
Change
let dispositionSchema = new mongoose.Schema({
name : {type: String, default: null},
mobile : {type : String, default: null},
remarks : {type: String, default:null},
duration: {type : String, default: null},
amount : {type : Number, default: 0},
date : {type : String, default: null},
time : {type : String, default: null},
createdAt: {type: Date, default: Date.now },
updatedAt: {type: Date, default: Date.now }
});
To
let dispositionSchema = new mongoose.Schema({
name : {type: String, default: null},
mobile : {type : String, default: null},
remarks : {type: String, default:null},
duration: {type : String, default: null},
amount : {type : Number, default: 0},
date : {type : String, default: null},
time : {type : String, default: null},
createdAt: {type: Date, default: Date.now },
updatedAt: {type: Date, default: Date.now }
}, { timestamps: { createdAt: 'createdAt' , updatedAt: 'updatedAt'} });
you can now set a timestamps option on the Schema to have Mongoose handle this for you:
var dispositionSchema = new Schema({..}, { timestamps: true });