How do I copy all objects from one prefix to other? I have tried all possible ways to copy all objects in one shot from one prefix to other, but the only way that seems to work is by looping over a list of objects and copying them one by one. This is really inefficient. If I have hundreds of files in a folder, will I have to make 100 calls?
var params = {
Bucket: bucket,
CopySource: bucket+'/'+oldDirName+'/filename.txt',
Key: newDirName+'/filename.txt',
};
s3.copyObject(params, function(err, data) {
if (err) {
callback.apply(this, [{
type: "error",
message: "Error while renaming Directory",
data: err
}]);
} else {
callback.apply(this, [{
type: "success",
message: "Directory renamed successfully",
data: data
}]);
}
});
You will need to make one AWS.S3.listObjects() to list your objects with a specific prefix. But you are correct in that you will need to make one call for every object that you want to copy from one bucket/prefix to the same or another bucket/prefix.
You can also use a utility library like async to manage your requests.
var AWS = require('aws-sdk');
var async = require('async');
var bucketName = 'foo';
var oldPrefix = 'abc/';
var newPrefix = 'xyz/';
var s3 = new AWS.S3({params: {Bucket: bucketName}, region: 'us-west-2'});
var done = function(err, data) {
if (err) console.log(err);
else console.log(data);
};
s3.listObjects({Prefix: oldPrefix}, function(err, data) {
if (data.Contents.length) {
async.each(data.Contents, function(file, cb) {
var params = {
Bucket: bucketName,
CopySource: bucketName + '/' + file.Key,
Key: file.Key.replace(oldPrefix, newPrefix)
};
s3.copyObject(params, function(copyErr, copyData){
if (copyErr) {
console.log(copyErr);
}
else {
console.log('Copied: ', params.Key);
cb();
}
});
}, done);
}
});
Hope this helps!
Here is a code snippet that do it in the "async await" way:
const AWS = require('aws-sdk');
AWS.config.update({
credentials: new AWS.Credentials(....), // credential parameters
});
AWS.config.setPromisesDependency(require('bluebird'));
const s3 = new AWS.S3();
... ...
const bucketName = 'bucketName'; // example bucket
const folderToMove = 'folderToMove/'; // old folder name
const destinationFolder = 'destinationFolder/'; // new destination folder
try {
const listObjectsResponse = await s3.listObjects({
Bucket: bucketName,
Prefix: folderToMove,
Delimiter: '/',
}).promise();
const folderContentInfo = listObjectsResponse.Contents;
const folderPrefix = listObjectsResponse.Prefix;
await Promise.all(
folderContentInfo.map(async (fileInfo) => {
await s3.copyObject({
Bucket: bucketName,
CopySource: `${bucketName}/${fileInfo.Key}`, // old file Key
Key: `${destinationFolder}/${fileInfo.Key.replace(folderPrefix, '')}`, // new file Key
}).promise();
await s3.deleteObject({
Bucket: bucketName,
Key: fileInfo.Key,
}).promise();
})
);
} catch (err) {
console.error(err); // error handling
}More update on the original code which copies folders recursively. Some limitations is that the code does not handle more than 1000 objects per Prefix and of course the depth limitation if your folders are very deep.
import AWS from 'aws-sdk';
AWS.config.update({ region: 'ap-southeast-1' });
/**
* Copy s3 folder
* @param {string} bucket Params for the first argument
* @param {string} source for the 2nd argument
* @param {string} dest for the 2nd argument
* @returns {promise} the get object promise
*/
export default async function s3CopyFolder(bucket, source, dest) {
// sanity check: source and dest must end with '/'
if (!source.endsWith('/') || !dest.endsWith('/')) {
return Promise.reject(new Error('source or dest must ends with fwd slash'));
}
const s3 = new AWS.S3();
// plan, list through the source, if got continuation token, recursive
const listResponse = await s3.listObjectsV2({
Bucket: bucket,
Prefix: source,
Delimiter: '/',
}).promise();
// copy objects
await Promise.all(
listResponse.Contents.map(async (file) => {
await s3.copyObject({
Bucket: bucket,
CopySource: `${bucket}/${file.Key}`,
Key: `${dest}${file.Key.replace(listResponse.Prefix, '')}`,
}).promise();
}),
);
// recursive copy sub-folders
await Promise.all(
listResponse.CommonPrefixes.map(async (folder) => {
await s3CopyFolder(
bucket,
`${folder.Prefix}`,
`${dest}${folder.Prefix.replace(listResponse.Prefix, '')}`,
);
}),
);
return Promise.resolve('ok');
}