Amazon S3 Remote File Upload With Axios

I am trying to write a function that would: Take a remote URL as a parameter, Get the file using axios Upload the stream to amazon s3 And finally, return the uploaded url I found

Solution 1:

After an upload you can call the getsignedurl function in s3 sdk to get the url where you can also specify the expiry of the url as well. You need to pass the key for that function. Now travelling will update with example later.

To generate a simple pre-signed URL that allows any user to view the contents of a private object in a bucket you own, you can use the following call to getSignedUrl():

 var s3 = new AWS.S3(); 
 var params = {Bucket: 'myBucket', Key: 'myKey'}; 
 s3.getSignedUrl('getObject', params, function (err, url) {  
   console.log("The URL is", url); 
 });

Official documentation link http://docs.amazonaws.cn/en_us/AWSJavaScriptSDK/guide/node-examples.html

Code must be something like this

function uploadFileToS3AndGenerateUrl(cb) {
const pass = new stream.PassThrough();//I have generated streams from file. Using this since this is what you have used. Must be a valid one.
var params = {
            Bucket: "your-bucket", // required
            Key: key , // required
            Body: pass,
            ContentType: 'your content type',

        };
s3.upload(params, function(s3Err, data) {
    if (s3Err) {
        cb(s3Err)
    }
    console.log(`File uploaded successfully at ${data.Location}`)

    const params = {
        Bucket: 'your-bucket',
        Key: data.key,
        Expires: 180
    };
    s3.getSignedUrl('getObject', params, (urlErr, urlData) => {
        if (urlErr) {

            console.log('There was an error getting your files: ' + urlErr);
            cb(urlErr);

        } else {
            console.log(`url: ${urlData}`);
            cb(null, urlData);

        }
    })
})
}

Solution 2:

Please check i have update your code might its help you.

    /*
         * Method to upload remote file to s3
         */
        const uploadRemoteFileToS3 = async (remoteAddr) => {
            const response = await axios({
                method: 'get',
                url: remoteAddr,
                responseType: 'stream'
            })
               if(response.status===200){
                    const file_name = remoteAddr.substring(remoteAddr.lastIndexOf('/')+1);
                    const content_type = response.headers['content-type'];
                    response.data.pipe(uploadFromStream(file_name, content_type));
                }
                return new Promise((resolve, reject) => {
                    response.data.on('end', (response) => {
                      console.log(response)
                      resolve(response)
                    })

                    response.data.on('error', () => {
                      console.log(response);
                      reject(response)
                    })
              })
        };

       * 
     * Method to pipe the stream 
     */
    const uploadFromStream = (file_name, content_type) => {
       return new Promise((resolve, reject) => {
          const pass = new stream.PassThrough();
          const obj_key = generateObjKey(file_name);
          const params = { Bucket: config.bucket, ACL: config.acl, Key: obj_key, ContentType: content_type, Body: pass };
          s3.upload(params, function(err, data) {
            if(!err){
                console.log(data)
                return resolve(data.Location);
            } else {
                console.log(err)
                return reject(err);
            }
          });
       });
    }

//call uploadRemoteFileToS3
    uploadRemoteFileToS3(remoteAddr)
      .then((finalResponse) => {
            console.log(finalResponse)
       })
       .catch((err) => {
         console.log(err);
    });