It doesn't look to me that using python to combine all the json files is convenient, and the combined file would be 30G.
I am using mongoDB and nodejs. The way how I populate a sample json is:
var data = require('./data1.json')
var populateDB = function() {
db.collection('temp', function(err, collection) {
collection.insert(data, {safe:true}, function(err, result) {});
});
};
This only adds one json file. How should I populate the collection with the 10000+ json files from here? any suggestion is highly appreciated!
The easiest way would be to write a Node program that processes one JSON file and then run it multiple time from the shell:
for i in *.json; do node program.js $i; done
Your Node program would just need to access the name from process.argv instead of having it hardcoded but the logic would be the same.
If you want to do everything in node then you will have to read the directory, get all .json files, read every one of them in sequence and then run a code similar to the one you posted. If it's a one off task then you can even get away with using the "Sync" functions to simplify your code if it's a sequential task to do one thing at a time and you don't care about adding the data in parallel.
Something like this would work
npm i glob-fs mongodb async --save
const async = require('async');
const fs = require('fs');
const glob = require('glob-fs')({ gitignore: true });
const MongoClient = require('mongodb').MongoClient;
const files = './files/data*.json';
const collection = 'test';
const url = 'mongodb://localhost:27017/test';
// Connect to db
MongoClient.connect(url, function (err, db) {
if (err) {
console.log(err);
}
// Get the collection
const col = db.collection(collection);
glob.readdirPromise(files)
.then(function (f) {
return async.eachSeries(f, (item, callback) => {
fs.readFile(item, 'utf8', function (err, data) {
if (err) {
return console.log(err);
}
// Insert into mongo
col.insertMany(JSON.parse(data)).then((r) => {
console.log(r);
return callback(r);
}).catch(function (fail) {
console.log(fail)
});
});
}, err => {
console.log(err);
});
})
.then(err => {
if (err) {
db.close();
}
})
.catch(err => {
console.log(err);
});
});