This is driving me nuts.
I need my server to download 100 relatively small files files from S3 (max 2 Mb).
It always work for ~95% of the files, but it blocks for the last 6-8 files. The resolve nor the reject callback gets never called...
I tried to download files in parallel ...
Here's the code, which technically works in 95% of the cases :
let singleGetFromS3 = (bucket, fileName) => {
return new Promise((resolve, reject) => {
let extension = getFileNameExtension(fileName);
fs.stat(`./${fileName}`, (err, stat) => {
if (err === null) {
console.log(`${fileName} exists locally`);
resolve(fileName);
} else if(err.code === 'ENOENT') {
let params = {Bucket: bucket, Key: fileName};
let file = require('fs').createWriteStream(`./tmp-${fileName}`);
s3.getObject(params).createReadStream()
.on('error', (error) => { return reject(error); })
.on('end', () => {
fs.rename(`./tmp-${fileName}`, `./${fileName}`, reject);
return resolve(fileName);
})
.pipe(file);
} else {
reject(err);
}
});
});
};
Using:
"aws-sdk": "^2.23.0",
node --version
v4.5.0
Looks like you are listening for the wrong event for when to resolve. You don't want to resolve when all bytes have been read from S3's Readable, you want to resolve when file is done writing.
let singleGetFromS3 = (bucket, fileName) => {
return new Promise((resolve, reject) => {
let extension = getFileNameExtension(fileName);
fs.stat(`./${fileName}`, (err, stat) => {
if (err === null) {
console.log(`${fileName} exists locally`);
resolve(fileName);
} else if(err.code === 'ENOENT') {
let params = {Bucket: bucket, Key: fileName};
let file = require('fs').createWriteStream(`./tmp-${fileName}`);
// Listen for the file to be done writing, then resolve
file.on('finish', () => {
fs.rename(`./tmp-${fileName}`, `./${fileName}`, reject);
return resolve(fileName);
})
s3.getObject(params).createReadStream()
.on('error', (error) => { return reject(error); })
.pipe(file);
} else {
reject(err);
}
});
});
};