I'm trying to grab thousands of files from Amazon S3 within a Promise but I can't seem to figure out how to include the ContinuationToken within if the list is truncated and gather it all together within the promise. I'm a novice with JS and could use some help. Here's what I have, so far:
getFiles()
.then(filterFiles)
.then(mapUrls)
;
function getFiles(token) {
var params = {
Bucket: bucket,
MaxKeys: 5000,
ContinuationToken: token
};
var allKeys = [];
var p = new Promise(function(resolve, reject){
s3.listObjectsV2(params, function(err, data) {
if (err) {
return reject(err);
}
allKeys.push(data.Contents)
if (data.IsTruncated) {
s3.listObjectsV2({Bucket: bucket, MaxKeys: 5000, ContinuationToken: data.NextContinuationToken})
console.log('Getting more images...');
allKeys.push(data.Contents)
}
resolve(data.Contents);
});
});
return p;
}
I need the function to continue to run until I've created a list of all objects in the bucket to return.
You need ContinuationToken the second time only.
var params = {
Bucket: bucket,
MaxKeys: 5000,
};
if (data.IsTruncated) {
s3.listObjectsV2({...params, ContinuationToken: data.NextContinuationToken})
IMO, this is just a s3 function called twice, more like a nested call. Recursion is when a function keeps calling itself until a specified condition is met.
Read more about recursion: https://medium.com/@vickdayaram/recursion-caad288bf621
I was able to list all objects in the bucket using async/await and the code below to populate an array.
async function getFiles(objects = []) {
const response = await s3.listObjectsV2(params).promise();
response.Contents.forEach(obj => filelist.push(obj.Key));
if (response.NextContinuationToken) {
params.ContinuationToken = response.NextContinuationToken;
await getFiles(params, objects);
}
console.log(filelist.length)
return filelist;
}
Thanks to all who helped!