Path to a local file, either relative to cwd, or absolute
+The resolved CacheControl value based on the provided settings
+Based on the local path and the provided glob pattern, this util function will find all relevant + files, which will be used to upload in another step.
+A list of resolved files based on the glob pattern
Executes the upload operation based on the provided options in the Uploader constructor.
+A list of paths of the upload files relative to the bucket.
Uploads a single file to S3 from the local to the remote path with the available options, + and returns the uploaded location.
+Path to the local file, either relative to cwd, or absolute
+The path to upload the file to in the bucket
+The remote path upload location relative to the bucket
import Uploader from 's3-batch-upload';
+ Usage: cli.js <command> [options]
-await new Uploader({
- config: './config/configS3.json', // can also use environment variables
+Commands:
+ cli.js upload Upload files to s3 [default]
+
+Required:
+ -b, --bucket The bucket to upload to. [string] [required]
+ -p, --local-path The path to the folder to upload. [string] [required]
+ -r, --remote-path The remote path in the bucket to upload the files to. [string] [required]
+
+Options:
+ -d, --dry-run Do a dry run, don't do any upload. [boolean] [default: false]
+ -C, --concurrency The amount of simultaneous uploads, increase on faster internet connection.
+ [number] [default: 100]
+ -g, --glob A glob on filename level to filter the files to upload [string] [default: "*.*"]
+ -a, --cache-control Cache control for uploaded files, can be string for single value or list of glob settings
+ [string] [default: ""]
+ -acl, --access-control-level Sets the access control level for uploaded files
+ [string] [default: "undefined"]
+ -c, --config The AWS config json path to load S3 credentials with loadFromPath. [string]
+ -h, --help Show help [boolean]
+
+Examples:
+ cli.js -b bucket-name -p ./files -r /data Upload files from a local folder to a s3 bucket path
+ cli.js ... -a "max-age=300" Set cache-control for all files
+ cli.js ... -a '{ "**/*.json": "max-age=300", "**/*.*": Upload files from a local folder to a s3 bucket path
+ "max-age=3600" }'
+ cli.js -d ... Dry run upload
+
+for more information about AWS authentication, please visit
+https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-credentials-node.html
+
API
+ import Uploader from 's3-batch-upload';
+
+const files = await new Uploader({
+ config: './config/configS3.json', // can also use environment variables
bucket: 'bucket-name',
- localPath: './files',
- remotePath: 'remote/path/in/bucket',
- glob: '*.jpg', // default is '*.*'
+ localPath: './files',
+ remotePath: 'remote/path/in/bucket',
+ glob: '*.jpg', // default is '*.*'
concurrency: '200', // default is 100
dryRun: true, // default is false
+ cacheControl: 'max-age=300', // can be a string, for all uploade resources
+ cacheControl: { // or an object with globs as keys to match the input path
+ '**/settings.json': 'max-age=60', // 1 mins for settings, specific matches should go first
+ '**/*.json': 'max-age=300', // 5 mins for other jsons
+ '**/*.*': 'max-age=3600', // 1 hour for everthing else
+ },
+ accessControlLevel: 'bucket-owner-full-control' // optional, not passed if undefined. - available options - "private"|"public-read"|"public-read-write"|"authenticated-read"|"aws-exec-read"|"bucket-owner-read"|"bucket-owner-full-control"
}).upload();
+
+// the files array contains a list of uploaded keys, which you can use to build up the S3 urls.
+// e.g. "remote/path/in/bucket/demo.jpg"
+
+ S3 Authentication
+ As seem above, you can either use environment variables, or a config file.
+ When using a config file, add it to the .gitignore, because you don't want your credentials
+ in your repo. Use the following template for the config file as stated in the AWS Docs:
+ {
+ "accessKeyId": "<YOUR_ACCESS_KEY_ID>",
+ "secretAccessKey": "<YOUR_SECRET_ACCESS_KEY>",
+ "region": "us-east-1"
+}
+ When using environment variables, check the AWS docs.
Documentation
View the generated documentation.
Building
@@ -160,6 +212,9 @@ LICENSE
mime
+
+ minimatch
+
ora
@@ -169,7 +224,7 @@ LICENSE
yargs
-
+
streamBatch
diff --git a/src/lib/Uploader.ts b/src/lib/Uploader.ts
index 46f3d67..a8d592f 100644
--- a/src/lib/Uploader.ts
+++ b/src/lib/Uploader.ts
@@ -53,11 +53,15 @@ export default class Uploader {
this.s3 = this.options.s3Client || new AWS.S3();
}
- public upload(): Promise {
+ /**
+ * Executes the upload operation based on the provided options in the Uploader constructor.
+ * @returns A list of paths of the upload files relative to the bucket.
+ */
+ public upload(): Promise {
return this.run();
}
- private async run(): Promise {
+ private async run(): Promise {
const files = await this.getFiles();
const { concurrency, localPath, remotePath } = this.options;
@@ -70,10 +74,10 @@ export default class Uploader {
});
// do the work!
- await streamBatch({
+ const results = await streamBatch({
files,
concurrency,
- processItem: (file: string): Promise => {
+ processItem: (file: string): Promise => {
const key = path.join(remotePath, file);
return this.uploadFile(path.resolve(localPath, file), key);
},
@@ -82,8 +86,15 @@ export default class Uploader {
// tslint:disable-next-line no-console
console.log('Upload complete!');
+
+ return results;
}
+ /**
+ * Based on the local path and the provided glob pattern, this util function will find all relevant
+ * files, which will be used to upload in another step.
+ * @returns A list of resolved files based on the glob pattern
+ */
private getFiles(): Promise> {
const { localPath, glob: globPath } = this.options;
const gatheringSpinner = ora(`Gathering files from ${chalk.blue(localPath)} (please wait) ...`);
@@ -106,7 +117,15 @@ export default class Uploader {
});
}
- public uploadFile(localFilePath: string, remotePath: string): Promise {
+ /**
+ * Uploads a single file to S3 from the local to the remote path with the available options,
+ * and returns the uploaded location.
+ *
+ * @param localFilePath Path to the local file, either relative to cwd, or absolute
+ * @param remotePath The path to upload the file to in the bucket
+ * @returns The remote path upload location relative to the bucket
+ */
+ public uploadFile(localFilePath: string, remotePath: string): Promise {
const body = fs.createReadStream(localFilePath);
const { dryRun, bucket: Bucket, accessControlLevel: ACL } = this.options;
const params: S3.PutObjectRequest = {
@@ -125,14 +144,19 @@ export default class Uploader {
this.s3.upload(params, err => {
// tslint:disable-next-line no-console
if (err) console.error('err:', err);
- resolve();
+ resolve(params.Key);
});
} else {
- resolve();
+ resolve(params.Key);
}
});
}
+ /**
+ *
+ * @param file Path to a local file, either relative to cwd, or absolute
+ * @return The resolved CacheControl value based on the provided settings
+ */
public getCacheControlValue(file: string): string {
const { cacheControl } = this.options;
if (cacheControl) {
diff --git a/src/lib/batch.ts b/src/lib/batch.ts
index 37d58e1..ff190d3 100644
--- a/src/lib/batch.ts
+++ b/src/lib/batch.ts
@@ -1,22 +1,24 @@
-export type Options = {
+export type Options = {
concurrency: number;
files: Array;
- processItem: (file: string) => Promise;
+ processItem: (file: string) => Promise;
onProgress: () => void;
};
-export default function streamBatch({
+export default function streamBatch({
concurrency,
files,
processItem,
onProgress,
-}: Options): Promise {
+}: Options): Promise {
return new Promise(resolve => {
let count = 0;
const total = files.length;
+ const results: T[] = [];
// when upload for one item is done, complete or process the next
- const onItemDone = () => {
+ const onItemDone = (result: T) => {
+ results.push(result);
count += 1;
// if completed
@@ -24,7 +26,7 @@ export default function streamBatch({
// temp fix for https://github.com/visionmedia/node-progress/pull/183
setTimeout(() => {
onProgress();
- resolve();
+ resolve(results);
}, 50);
} else {
onProgress();
diff --git a/test/Uploader.spec.ts b/test/Uploader.spec.ts
index 3b865b7..b0d817a 100644
--- a/test/Uploader.spec.ts
+++ b/test/Uploader.spec.ts
@@ -163,5 +163,62 @@ describe('Uploader', () => {
});
});
+ describe('with uploadFile', () => {
+ it('should return the uploaded path', async function() {
+ this.timeout(5000);
+
+ const s3 = {
+ upload(_, cb) {
+ cb(null);
+ }
+ };
+ spy(s3, "upload");
+
+ uploader = new Uploader({
+ localPath: 'test/files',
+ remotePath: 'fake',
+ bucket: 'fake',
+ glob: '**/demo.png',
+ s3Client: s3,
+ });
+
+ const result = await uploader.uploadFile('files/demo.png', 'foo\\bar.png');
+
+ expect(result).to.equal('foo/bar.png');
+
+ (s3.upload).restore();
+ });
+ });
+
+ describe('with uploadFile', () => {
+ it('should return the uploaded paths', async function() {
+ this.timeout(10000);
+
+ const s3 = {
+ upload(_, cb) {
+ cb(null);
+ }
+ };
+ spy(s3, "upload");
+
+ uploader = new Uploader({
+ localPath: 'test/files',
+ remotePath: 'fake',
+ bucket: 'fake',
+ glob: '**/demo.png',
+ s3Client: s3,
+ accessControlLevel: 'bucket-owner-full-control'
+ });
+
+ const results = await uploader.upload();
+
+ expect(results).to.deep.equal([
+ 'fake/demo.png'
+ ]);
+
+ (s3.upload).restore();
+ });
+ });
+
});
});
diff --git a/wallaby.js b/wallaby.js
new file mode 100644
index 0000000..eb1a3f7
--- /dev/null
+++ b/wallaby.js
@@ -0,0 +1,24 @@
+module.exports = function(wallaby) {
+
+ return {
+ files: [
+ 'tsconfig.json',
+ 'src/**/*.*',
+ 'test/**/*.*',
+ '!test/**/*.spec.ts',
+ ],
+
+ tests: ['test/**/*.spec.ts'],
+
+ env: {
+ type: 'node',
+ runner: 'node',
+ },
+
+ compilers: {
+ '**/*.ts': wallaby.compilers.babel()
+ },
+
+ testFramework: 'mocha',
+ };
+};