My file size is 2483385kb, the following code using Papa Parser does not help me:
this.readFile((output) => {
_this.sample = get(Papa.parse(output, {preview: 2, skipEmptyLines: true, chunkSize: 1024 * 1024 * 45}), "data");
_this.csv = get(Papa.parse(output, {skipEmptyLines: true, chunkSize: 1024 * 1024 * 45}), "data");
});
It cannot read the large csv file. Am I doing it wrongly?
You can try streaming.
/**
* Returns an async generator that yields the CSV file line-by-line.
*
* @param path {String}
* @param options {papa.ParseConfig}
*/
async function* readCsvStream(path, options) {
const csvStream = createReadStream(path);
const parseStream = papa.parse(papa.NODE_STREAM_INPUT, options);
csvStream.pipe(parseStream);
for await (const chunk of parseStream) {
yield chunk;
}
}
(async () => {
try {
const asyncIterator = readCsvStream(`${__dirname}/cad_fi.csv`, {
header: true,
dynamicTyping: true,
});
for await (const chunk of asyncIterator) {
// You can do anything with each row of the `.csv` file here.
// like saving it to a DB row by row
console.log(chunk);
}
} catch (error) {
console.error(error);
}
})();
Or, alternatively, you can pass a step callback to the parse function, which will be called on each line, thus avoiding loading the entire csv in memory.
/**
* Reads a CSV file.
*
* @param path {String}
* @param options {papa.ParseConfig}
*/
function readCsv(csvString, stepCallback, options) {
papa.parse(csvString, {
...options,
step: stepCallback,
});
}
(async () => {
// I read it from FS because it is NodeJs, but you can get the string by any means.
const csv = (await readFile(`${__dirname}/cad_fi.csv`)).toString();
const data = [];
const errors = [];
const stepCallback = (results, parser) => {
if (results?.errors?.length) {
errors.push(...results.errors);
}
data.push(results.data);
};
const papaparseOptions = {
header: true,
dynamicTyping: true,
};
try {
readCsv(
csv,
stepCallback,
papaparseOptions,
);
console.log(errors);
console.log(data.length);
} catch (error) {
console.error(error);
}
})();