I need to generate a huge number of combinations of string values, a few billion of lines and write them all in one large file to disk without the process being blocked.
At this moment it works until the file write buffer is filled, for little lists of words, when I try with a larger number of records the output files remains empty.
const fs = require('fs');
const words = fs.readFileSync('./1024words.txt').toString().split("\n");
//create array of 1024 words
let fileout
, row = ''
, wordA, wordB, wordC
, totRows = Math.pow(words.length,3)*3 //combination of 3 words for 3 columns of output
, count = 0;
for (let indexA = 0; indexA < words.length; indexA++) {
wordA = words[ indexA ];
fileout = fs.createWriteStream(`${wordA}.csv`); //new output file
for (let indexB = 0; indexB < words.length; indexB++) {
wordB = words[ indexB ];
for (let indexC = 0; indexC < words.length; indexC++) {
wordC = words[ indexC ];
row = `${wordA} ${wordB} ${wordC}\n`;
count++;
process.stdout.write(`${count} of ${totRows} written: ${row}`);
fileout.write(row);
}
}
fileout.end();
fileout.close();
}
you're missing a backpressure handling
further reading here
https://nodejs.org/en/docs/guides/backpressuring-in-streams/
add this function
function writeToStream(stream, data) {
const canContinue = stream.write(data);
if (!canContinue) {
stream.pause()
stream.once('drain', () => {
stream.resume()
})
}
}
full code :
const fs = require('fs');
const words = fs.readFileSync('./1024words.txt').toString().split("\n");
//create array of 1024 words
let fileout
, row = ''
, wordA, wordB, wordC
, totRows = Math.pow(words.length, 3) * 3 //combination of 3 words for 3 columns of output
, count = 0;
for (let indexA = 0; indexA < words.length; indexA++) {
wordA = words[indexA];
fileout = fs.createWriteStream(`${wordA}.csv`); //new output file
for (let indexB = 0; indexB < words.length; indexB++) {
wordB = words[indexB];
for (let indexC = 0; indexC < words.length; indexC++) {
wordC = words[indexC];
row = `${wordA} ${wordB} ${wordC}\n`;
count++;
writeToStream(process.stdout, `${count} of ${totRows} written: ${row}`)
writeToStream(fileout, row)
}
}
fileout.end();
fileout.close();
}
function writeToStream(stream, data) {
const canContinue = stream.write(data);
if (!canContinue) {
stream.pause()
stream.once('drain', () => {
stream.resume()
})
}
}