I'm extracting data from a csv file in to an array. I'm using the Streams approach to extract the data from the CSV into the the array. Once this Data has been extracted i need to do some processing on it using the reduce method. Here is where the problem starts. I run into a JavaScript heap out of memory Error.
I have managed to extract data without this error. How can I process this large data in a better way to avoid getting this error?
Here is my code
inputStream.pipe(new CsvReadableStream({
parseNumbers: true,
parseBooleans: true,
trim: true
}))
.on('data', (row) => {
csvData.push(row);
})
.on('end', function() {
groupedResult = csvData.reduce((acc, curr) => { //<- How can i Process this large data in a better way?
if (!acc[curr[2]]) {
acc[curr[2]] = [];
}
acc[curr[2]].push(curr);
return acc;
}, []);
for (let groupName in groupedResult) { //<- How can i Process this large data in a better way?
const accumulated = groupedResult[groupName].reduce((acc, curr) => {
acc[curr[1]] += curr[3];
return acc; // ← return the accumulator to use for the next round
}, {
DEPOSIT: 0,
WITHDRAWAL: 0
})
groupedBalance[groupName] = {
Balance: accumulated['DEPOSIT'] - accumulated['WITHDRAWAL']
};
}
for (let Gname in groupedBalance) {
axios.get('https://min-api.cryptocompare.com/data/price?fsym=' + Gname + '&tsyms=USD&api_key=a437b4c23456')
.then(response => {
groupedApiResult[Gname] = {
LastestBalance: ((groupedBalance[Gname].Balance * response.data.USD))
};
console.log(groupedApiResult);
})
.catch(error => {
console.log(error);
});
}
});