Business
Jobs
  • About Us
  • Solutions
    • Job Postings
      Post your job and receive qualified candidates in 48h.
    • Candidate Assessments
      500+ technical and psychological tests, plus anti-fraud.
    • Headhunting
      Tailor-made executive search from start to finish.
    • Payroll + EOR
      Payroll dispersal and EOR across 15+ LATAM countries.
  • Pricing
  • Jobs

0

1.3K
Views
Uploading multiple files to AWS S3 using NodeJS

I'm trying to upload all files within my directory to my S3 bucket using NodeJS. I'm able to upload one file at a time if I explicitly give the file path + literal string for the Key: field.

Below is the script I'm using:

var AWS = require('aws-sdk'),
    fs = require('fs');

// For dev purposes only
AWS.config.update({ accessKeyId: '...', secretAccessKey: '...' });

// reg ex to match
var re = /\.txt$/;

// ensure that this file is in the directory of the files you want to run the cronjob on

// ensure that this file is in the directory of the files you want to run the cronjob on
fs.readdir(".", function(err, files) {
    if (err) {
    console.log( "Could not list the directory.", err)
    process.exit( 1 )
    }


    var matches = files.filter( function(text) { return re.test(text) } )
    console.log("These are the files you have", matches)
    var numFiles = matches.length


    if ( numFiles ) {
        // Read in the file, convert it to base64, store to S3

        for( i = 0; i < numFiles; i++ ) {
            var fileName = matches[i]

            fs.readFile(fileName, function (err, data) {
                if (err) { throw err }

                // Buffer Pattern; how to handle buffers; straw, intake/outtake analogy
                var base64data = new Buffer(data, 'binary');


                var s3 = new AWS.S3()
                    s3.putObject({
                       'Bucket': 'noonebetterhaventakenthisbucketnname',
                        'Key': fileName,
                        'Body': base64data,
                        'ACL': 'public-read'
                     }, function (resp) {
                        console.log(arguments);
                        console.log('Successfully uploaded, ', fileName)
                    })
            })

        }

    }

})

It produces this error for each file attempted to upload to S3:

These are the files you have [ 'test.txt', 'test2.txt' ]
{ '0': null,
  '1': { ETag: '"2cad20c19a8eb9bb11a9f76527aec9bc"' } }
Successfully uploaded,  test2.txt
{ '0': null,
  '1': { ETag: '"2cad20c19a8eb9bb11a9f76527aec9bc"' } }
Successfully uploaded,  test2.txt

edit: updated using a variable name to allow key to be read instead of matches[i]

Why does it only upload test2.txt, and how do I get it to upload each file within my matches variable?

over 4 years ago · Santiago Trujillo
1 answers
Answer question

0

Referenced this Asynchronously reading and caching multiple files in nodejs to arrive at a solution.

tl;dr scope issue - need to wrap variables in closure; can do this by creating a function for the readFile and s3.putObject and calling that within the for loop.

var AWS = require('aws-sdk'),
    fs = require('fs');

// For dev purposes only
AWS.config.update({ accessKeyId: '...', secretAccessKey: '...' });

var s3 = new AWS.S3()

function read(file) {
    fs.readFile(file, function (err, data) {
        if (err) { throw err }

        // Buffer Pattern; how to handle buffers; straw, intake/outtake analogy
        var base64data = new Buffer(data, 'binary');

        s3.putObject({
           'Bucket': 'noonebetterhaventakenthisbucketnname',
            'Key': file,
            'Body': base64data,
            'ACL': 'public-read'
         }, function (resp) {
            console.log(arguments);
            console.log('Successfully uploaded, ', file)
        })
    })
}

// reg ex to match
var re = /\.txt$/;

// ensure that this file is in the directory of the files you want to run the cronjob on
fs.readdir(".", function(err, files) {
    if (err) {
        console.log( "Could not list the directory.", err)
        process.exit( 1 )
    }

    var matches = files.filter( function(text) { return re.test(text) } )
    console.log("These are the files you have", matches)
    var numFiles = matches.length


    if ( numFiles ) {
        // Read in the file, convert it to base64, store to S3

        for( i = 0; i < numFiles; i++ ) {
            read(matches[i])
        }

    }

})
over 4 years ago · Santiago Trujillo Report
Answer question
Find remote jobs

Discover the new way to find a job!

Top jobs
Top job categories
Business
Post vacancy Pricing Sales
Legal
Terms and conditions Privacy policy
© 2026 PeakU Inc. All Rights Reserved.
Andres GPT
Show me some job opportunities
There's an error!