Skip to content

Commit

Permalink
Return the list of uploaded files from upload and uploadFile
Browse files Browse the repository at this point in the history
  • Loading branch information
ThaNarie committed Jun 9, 2019
1 parent 12a6fe2 commit 7a50a25
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 14 deletions.
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/setting-creden
```js
import Uploader from 's3-batch-upload';

await new Uploader({
const files = await new Uploader({
config: './config/configS3.json', // can also use environment variables
bucket: 'bucket-name',
localPath: './files',
Expand All @@ -80,6 +80,9 @@ await new Uploader({
},
accessControlLevel: 'bucket-owner-full-control' // optional, not passed if undefined. - available options - "private"|"public-read"|"public-read-write"|"authenticated-read"|"aws-exec-read"|"bucket-owner-read"|"bucket-owner-full-control"
}).upload();

// the files array contains a list of uploaded keys, which you can use to build up the S3 urls.
// e.g. "remote/path/in/bucket/demo.jpg"
```

### S3 Authentication
Expand Down
38 changes: 31 additions & 7 deletions src/lib/Uploader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,15 @@ export default class Uploader {
this.s3 = this.options.s3Client || new AWS.S3();
}

public upload(): Promise<void> {
/**
* Executes the upload operation based on the provided options in the Uploader constructor.
* @returns A list of paths of the upload files relative to the bucket.
*/
public upload(): Promise<string[]> {
return this.run();
}

private async run(): Promise<void> {
private async run(): Promise<string[]> {
const files = await this.getFiles();
const { concurrency, localPath, remotePath } = this.options;

Expand All @@ -70,10 +74,10 @@ export default class Uploader {
});

// do the work!
await streamBatch({
const results = await streamBatch({
files,
concurrency,
processItem: (file: string): Promise<void> => {
processItem: (file: string): Promise<string> => {
const key = path.join(remotePath, file);
return this.uploadFile(path.resolve(localPath, file), key);
},
Expand All @@ -82,8 +86,15 @@ export default class Uploader {

// tslint:disable-next-line no-console
console.log('Upload complete!');

return results;
}

/**
* Based on the local path and the provided glob pattern, this util function will find all relevant
* files, which will be used to upload in another step.
* @returns A list of resolved files based on the glob pattern
*/
private getFiles(): Promise<Array<string>> {
const { localPath, glob: globPath } = this.options;
const gatheringSpinner = ora(`Gathering files from ${chalk.blue(localPath)} (please wait) ...`);
Expand All @@ -106,7 +117,15 @@ export default class Uploader {
});
}

public uploadFile(localFilePath: string, remotePath: string): Promise<void> {
/**
* Uploads a single file to S3 from the local to the remote path with the available options,
* and returns the uploaded location.
*
* @param localFilePath Path to the local file, either relative to cwd, or absolute
* @param remotePath The path to upload the file to in the bucket
* @returns The remote path upload location relative to the bucket
*/
public uploadFile(localFilePath: string, remotePath: string): Promise<string> {
const body = fs.createReadStream(localFilePath);
const { dryRun, bucket: Bucket, accessControlLevel: ACL } = this.options;
const params: S3.PutObjectRequest = {
Expand All @@ -125,14 +144,19 @@ export default class Uploader {
this.s3.upload(params, err => {
// tslint:disable-next-line no-console
if (err) console.error('err:', err);
resolve();
resolve(params.Key);
});
} else {
resolve();
resolve(params.Key);
}
});
}

/**
*
* @param file Path to a local file, either relative to cwd, or absolute
* @return The resolved CacheControl value based on the provided settings
*/
public getCacheControlValue(file: string): string {
const { cacheControl } = this.options;
if (cacheControl) {
Expand Down
14 changes: 8 additions & 6 deletions src/lib/batch.ts
Original file line number Diff line number Diff line change
@@ -1,30 +1,32 @@
export type Options = {
export type Options<T> = {
concurrency: number;
files: Array<string>;
processItem: (file: string) => Promise<void>;
processItem: (file: string) => Promise<T>;
onProgress: () => void;
};

export default function streamBatch({
export default function streamBatch<T>({
concurrency,
files,
processItem,
onProgress,
}: Options): Promise<void> {
}: Options<T>): Promise<T[]> {
return new Promise(resolve => {
let count = 0;
const total = files.length;
const results: T[] = [];

// when upload for one item is done, complete or process the next
const onItemDone = () => {
const onItemDone = (result: T) => {
results.push(result);
count += 1;

// if completed
if (!files.length && count === total) {
// temp fix for https://github.com/visionmedia/node-progress/pull/183
setTimeout(() => {
onProgress();
resolve();
resolve(results);
}, 50);
} else {
onProgress();
Expand Down
57 changes: 57 additions & 0 deletions test/Uploader.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,5 +163,62 @@ describe('Uploader', () => {
});
});

describe('with uploadFile', () => {
it('should return the uploaded path', async function() {
this.timeout(5000);

const s3 = {
upload(_, cb) {
cb(null);
}
};
spy(s3, "upload");

uploader = new Uploader({
localPath: 'test/files',
remotePath: 'fake',
bucket: 'fake',
glob: '**/demo.png',
s3Client: <any>s3,
});

const result = await uploader.uploadFile('files/demo.png', 'foo\\bar.png');

expect(result).to.equal('foo/bar.png');

(<any>s3.upload).restore();
});
});

describe('with uploadFile', () => {
it('should return the uploaded paths', async function() {
this.timeout(10000);

const s3 = {
upload(_, cb) {
cb(null);
}
};
spy(s3, "upload");

uploader = new Uploader({
localPath: 'test/files',
remotePath: 'fake',
bucket: 'fake',
glob: '**/demo.png',
s3Client: <any>s3,
accessControlLevel: 'bucket-owner-full-control'
});

const results = await uploader.upload();

expect(results).to.deep.equal([
'fake/demo.png'
]);

(<any>s3.upload).restore();
});
});

});
});

0 comments on commit 7a50a25

Please sign in to comment.