I am trying to extract multiple files from AWS S3 bucket and willing to merge the response from all files after.
E.g I have following files:
my-bucket/mainfile1.json.gz
my-bucket/mainfile2.json.gz
my-bucket/mainfile3.json.gz
Currently I am accessing a single file like this:
const unzipFromS3 = (key, bucket) => {
return new Promise(async (resolve, reject) => {
AWS.config.loadFromPath(process.env["PWD"]+'/private/awss3/s3_config.json');
var s3 = new AWS.S3();
let options = {
'Bucket': "my-bucket",
'Key': "mainfile1.json.gz",
};
s3.getObject(options, function(err, res) {
if(err) return reject(err);
resolve(zlib.unzipSync(res.Body).toString());
});
});
};
unzipFromS3().then(function(result){
console.dir(result);
});
Now this works perfect for single file, but how can I achieve this with multiple files in case I want to merge data from 3 separate files?
Here's an initial idea of how to read the gzipped JSON files from S3, unzip them, then merge the resulting JavaScript objects, and finally gzip and write the merged results back to S3.
const aws = require('aws-sdk');
const zlib = require('zlib');
const s3 = new aws.S3();
const BUCKET = 'mybucket';
const PREFIX = '';
const FILES = ['test1.json.gz', 'test2.json.gz', 'test3.json.gz'];
(async () => {
const promises = [];
try {
for (let ii = 0; ii < FILES.length; ii++) {
const params = {
Bucket: BUCKET,
Key: `${PREFIX}${FILES[ii]}`,
};
console.log('Get:', params.Key, 'from:', params.Bucket);
promises.push(s3.getObject(params).promise());
}
const results = await Promise.all(promises);
const buffers = results.map(result => result.Body);
const content = buffers.map(buffer => JSON.parse(zlib.unzipSync(buffer).toString()));
console.log('Read OK', JSON.stringify(content));
const merged = Object.assign({}, ...content);
console.log('Merged content', JSON.stringify(merged));
const params = {
Bucket: BUCKET,
Key: `${PREFIX}result/test.json.gz`,
Body: zlib.gzipSync(JSON.stringify(merged), 'utf8'),
};
console.log('Put:', params.Key, 'to:', params.Bucket);
const rc = await s3.putObject(params).promise()
} catch (err) {
console.log(err, err.stack);
throw err;
}
})();