I am writing a NodeJS script that reads this huge
and I need to write new file that contains only rows of data that have valid numeric profit values and then order the data based on profit value, and
I've used the mapHeaders option to convert the column headings to lowercase, and just use the first word (so we discard "(in millions)").
Then if the profit column exists, I parse it to a float. Then we can later use that in sort() to get the top 20 rows by profit.
Instead of saving all the rows in results, I just increment a variable containing a line counter.
const fs = require("fs");
const csv = require("csv-parser");
let lines = 0;
const validResults = [];
const inputFile = "./data.csv";
fs.createReadStream(inputFile)
.pipe(csv({
mapHeaders: h => h.split(" ")[0].toLowerCase(),
}))
.on("data", (data) => {
try {
if (data.profit) {
data.profit = parseFloat(data.profit);
validResults.push(data);
}
lines++;
} catch (err) {
console.log(err)
}
})
.on("end", () => {
console.log('# of Rows of data is', lines);
fs.writeFile('data2.json', JSON.stringify(validResults.sort((a, b) => b.profit - a.profit).slice(0, 20)), (err) => {
if (err) throw err;
console.log("File created")
})
});