EvaporateJS is a JavaScript library for uploading files to a S3 bucket using multipart uploads. You can pause/resume or cancel an upload.

I’ve tested it with Minio before moving to AWS or Scaleway.

Prerequisites

Install dependencies

You have to install the following applications :

You can use brew or any other package manager following you’re running on MacOS or Linux.

Start Minio

Once Minio is installed you can start it with the following command :

#!/bin/sh
export MINIO_ACCESS_KEY='<S3_ACCESS_KEY>'
export MINIO_SECRET_KEY='<S3_SECRET_KEY>'
export MINIO_REGION_NAME='us-east-1'

minio server --address '127.0.0.1:80' /path/to/data

Note: Minio is started on port 80 to avoid a bug with S3 signature

To get more info about the setup please read the Minio documentation

The architecture

Evaporate needs a specific endpoint to sign its HTTP request with the secret key before sending it to the S3 bucket

The GitHub repository includes many examples of endpoint implemented in Python, JavaScript, Java or Go.

Generating the S3 signature with GO

You must generate a signature for your S3 request before sending it. The method is documented here.

This work is delegated to a sign server. The GitHub repository of EvaporateJS contains many examples implemented in Python, JS, Java or Go. I’ve choose go for my sample.

package main

import (
	"crypto/hmac"
	"crypto/sha256"
	"fmt"
	"log"
	"net/http"
	"strings"
)

var (
    date string
    regionName string
    serviceName string
    requestName string
)

// Replace <S3_SECRET> by the secret associated to your access key
// If you're using minio this is the value of env variable MINIO_SECRET_KEY
const AWS_SECRET string = "<S3_SECRET>"

func HMAC(key, data []byte) []byte {
    h := hmac.New(sha256.New, key)
    h.Write(data)
    return h.Sum(nil)
}

func derivedKey(t string) []byte {
    h := HMAC([]byte("AWS4"+AWS_SECRET), []byte(t))
    h = HMAC(h, []byte(regionName))
    h = HMAC(h, []byte(serviceName))
    h = HMAC(h, []byte(requestName))
    return h
}

func signature(t, sts string) string {
    h := HMAC(derivedKey(t), []byte(sts))
    return fmt.Sprintf("%x", h)
}

func main() {
    http.HandleFunc("/sign_auth", func(rw http.ResponseWriter, r *http.Request) {
        qs := r.URL.Query()
        strs := strings.Split(qs.Get("to_sign"), "\n")
        data := strings.Split(strs[2], "/")

        date, regionName, serviceName, requestName = data[0], data[1], data[2], data[3]

        signedKey := signature(date, qs.Get("to_sign"))

        rw.Header().Add("Access-Control-Allow-Origin", "*")
        rw.Write([]byte(signedKey))
    });
    log.Fatal(http.ListenAndServe(":8080", nil))
}

Note: This is a really basic endpoint implementation you should add some security before deploying it in production.

Using Evaporate.js into your JavaScript application

For this section, I’ve created a sample application. Here is the content of file package.json:

{
  "name": "s3-upload",
  "version": "0.0.1",
  "description": "Sample application for upload files to S3 with Evaporate.JS",
  "private": true,
  "scripts": {
    "build": "webpack",
    "start": "webpack serve"
  },
  "author": "Mickaël GREGORI",
  "license": "ISC",
  "devDependencies": {
    "@babel/cli": "^7.12.1",
    "@babel/core": "^7.12.3",
    "@babel/preset-env": "^7.12.1",
    "babel-loader": "^8.1.0",
    "clean-webpack-plugin": "^3.0.0",
    "css-loader": "^5.0.0",
    "dotenv": "^8.2.0",
    "html-webpack-plugin": "^4.5.0",
    "style-loader": "^2.0.0",
    "webpack": "^5.2.0",
    "webpack-cli": "^4.1.0",
    "webpack-dev-server": "^3.11.0"
  },
  "dependencies": {
    "bootstrap": "^4.5.3",
    "evaporate": "^2.1.4",
    "jquery": "^3.5.1",
    "js-sha256": "^0.9.0",
    "popper.js": "^1.16.1",
    "spark-md5": "^3.0.1"
  }
}

You may noticed that I use webpack to run the application. The content of webpack configuration file is :

const path = require('path');
const webpack = require('webpack');
const HtmlWebpackPlugin = require('html-webpack-plugin');
const { CleanWebpackPlugin } = require('clean-webpack-plugin');
const dotenv = require('dotenv');

module.exports = {
	mode: 'development',
    entry: './src/index.js',
	devtool: 'inline-source-map',
	devServer: {
		contentBase: path.join(__dirname, 'dist'),
        port: 3000,
        hot: true
	},
	plugins: [
        new CleanWebpackPlugin({ cleanStaleWebpackAssets: false }),
		new HtmlWebpackPlugin({
			title: 'Development',
            template: 'index.html'
		}),
        new webpack.DefinePlugin({
            'process.env': JSON.stringify(dotenv.config().parsed)
        }),
    ],
	output: {
		filename: 'main.js',
		path: path.resolve(__dirname, 'dist'),
	},
        module: {
            rules: [
                { test: /\.css$/i, use: ['style-loader', 'css-loader'] },
                { test: /\.m?js$/, exclude: /node_modules/, use: { loader: 'babel-loader', options: { presets: ['@babel/preset-env'] } } }
            ]
        }
};

dotenv is required to configure our sample application. Evaporate needs your access key, the region and the name of your S3 bucket. More over the URL of the sign server is also required. All these information are put into a file named .env and loaded by dotenv.

Here is the content of file .env:

NODE_ENV=development
SIGNER_URL=<SIGN_SERVER_URL>
AWS_ACCESS_KEY=<S3_ACCESS_KEY>
AWS_S3_BUCKET_NAME=<S3_BUCKET_NAME>
AWS_URL=<S3_BUCKET_URL>

1- Start the sign server

To start the sign server run the following command

$ go run main.go

2- Build and start the web application

First we need a HTML file. Here is the content of index.html:

<!DOCTYPE html>
<html>
    <head>
        <title>S3 Upload</title>
    </head>
    <body>
        <main class="m-3">
            <section id="upload-form">
                <form>
                    <input id="file-upload" type="file" name="file"></input>
                    <button id="upload" name="upload">Upload</button>
                </form>
            </section>
            <div id="drag-zone" class="d-block p-3 mt-2 w-50 h-50 bg-warning shadow rounded">
               Drag &amp; drop files here 
            </div>
        </main>
    </body>
</html>

There is a form where you can select your file and click on a button to upload it. There is also a drag and drop zone.

Whatever the technique you choose the function uploadFile will be called. Here is the content of file index.js:

import Evaporate from 'evaporate';
import sparkMD5 from 'spark-md5';
import sha256 from 'js-sha256';
import 'bootstrap/dist/css/bootstrap.css';
import 'bootstrap';
import * as $ from 'jquery';

const { SIGNER_URL, AWS_ACCESS_KEY, AWS_S3_BUCKET_NAME, AWS_URL } = process.env; 

const uploader = Evaporate.create({
    signerUrl: SIGNER_URL || '/auth/signv4_upload',
    aws_url: AWS_URL || 'http://127.0.0.1',
    aws_key: AWS_ACCESS_KEY || 'AWS_PUBLIC_KEY',
    bucket: AWS_S3_BUCKET_NAME || 'videos',
    cloudfront: false,
    sendCanonicalRequestToSignerUrl: false,
    computeContentMd5: true,
    cryptoMd5Method: (d) => btoa(sparkMD5.ArrayBuffer.hash(d, true)),
    cryptoHexEncodedHash256: sha256,
});

function onUploadProgress(percent, stats) {
    // replace arrow function by a regular one because of this keyword 
    const filename = this.name;
    console.log('Progress', filename, percent, stats);
};

const onUploadComplete = (xhr, awsObjectKey) => {
    console.log('Complete!', awsObjectKey);
};

const onUploadError = (mssg) => {
    console.log('Error', mssg);
};

const onUploadPaused = () => {
    console.log('Paused');
};

const onUploadPausing = () => {
    console.log('Pausing');
};

const onUploadResumed = () => {
    console.log('Resumed');
};

const onUploadCancelled = () => {
    console.log('Cancelled');
};

const onUploadStarted = (fileKey) => {
    console.log('Started', fileKey);
};

const onUploadInitiated = (s3Id) => {
    console.log('Upload Initiated', s3Id);
};

const onUploadWarn = (mssg) => {
    console.log('File did not upload successfully:', reason);
};

const uploadFile = (file) => {
    console.log(file);
    uploader.then((evaporate) => {
        evaporate.add({
            file: file,
            name: file.name,
            progress: onUploadProgress,
            complete: onUploadComplete,
            error: onUploadError,
            paused: onUploadPaused,
            pausing: onUploadPausing,
            resumed: onUploadResumed,
            cancelled: onUploadCancelled,
            started: onUploadStarted,
            uploadInitiated: onUploadInitiated,
            warn: onUploadWarn
        }).then(
            (awsObjectKey) => console.log('File successfully uploaded to ', awsObjectKey),
            (reason) => console.log('File did not upload successfully: ',reason)
        );
    });
};

$('#drag-zone').on('dragenter', (e) => {
    e.stopPropagation();
    e.preventDefault();
});

$('#drag-zone').on('dragover', (e) => {
    e.stopPropagation();
    e.preventDefault();
});

$('#drag-zone').on('drop', (e) => {
    e.stopPropagation();
    e.preventDefault();
    const dt = e.originalEvent.dataTransfer;
    const files = dt.files;
    console.log(files);
    if (files) {
        Array.from(files).forEach(uploadFile);
    }
});

$('#upload').click( (e) => {
    e.preventDefault();
    let inputFiles = $("#upload-form form > input[type='file']");
    if (inputFiles && inputFiles.length) {
        let files = inputFiles[0].files;
        if (files.length) {
            Array.from(files).forEach(uploadFile);
        }
    }
    e.stopPropagation();
});

To start the application run the following command:

$ npm start

Great :) here we are :