From efdfabb9a479cad542abdf78a4f5c3fec9ebadd1 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Tue, 26 Jan 2021 16:57:20 -0700 Subject: [PATCH] v1.2.0: bugfix for buckets with invalid domain names, and pass request options - Buckets with names like 'example.bucket' and 'example_bucket' are valid as paths, but not as domain names. - allow passthrough support for latest @root/request, which supports pipes and streams --- README.md | 58 +++++++++++++++++++++++++++-- bin/s3-download.js | 56 ++++++++++++++-------------- index.js | 92 ++++++++++++++++++++++++++++++++++++---------- package-lock.json | 8 ++-- package.json | 5 ++- 5 files changed, 163 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index dc3115d..47a33cb 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,28 @@ A lightweight alternative to the S3 SDK that uses only @root/request and aws4. ### Download a file from S3 +This library supports the same streaming options as [@root/request.js](https://git.rootprojects.org/root/request.js). + +#### as a stream + ```js -s3.get({ +var resp = await s3.get({ + accessKeyId, // 'AKIAXXXXXXXXXXXXXXXX' + secretAccessKey, // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + region, // 'us-east-2' + bucket, // 'bucket-name' + prefix, // 'my-prefix/' (optional) + key, // 'data/stats.csv' (omits prefix, if any) + stream // fs.createWriteStream('./path/to/file.bin') +}); + +await resp.stream; +``` + +#### in-memory + +```js +var resp = await s3.get({ accessKeyId, // 'AKIAXXXXXXXXXXXXXXXX' secretAccessKey, // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' region, // 'us-east-2' @@ -22,12 +42,14 @@ s3.get({ prefix, // 'my-prefix/' (optional) key // 'data/stats.csv' (omits prefix, if any) }); + +fs.writeFile(resp.body, './path/to/file.bin'); ``` ### Upload a new file to S3 ```js -s3.set({ +await s3.set({ accessKeyId, secretAccessKey, region, @@ -41,6 +63,36 @@ s3.set({ }); ``` +### Check that a file exists + +```js +var resp = await s3.head({ + accessKeyId, // 'AKIAXXXXXXXXXXXXXXXX' + secretAccessKey, // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + region, // 'us-east-2' + bucket, // 'bucket-name' + prefix, // 'my-prefix/' (optional) + key // 'data/stats.csv' (omits prefix, if any) +}); + +console.log(resp.headers); +``` + +### Delete file + +```js +var resp = await s3.delete({ + accessKeyId, // 'AKIAXXXXXXXXXXXXXXXX' + secretAccessKey, // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + region, // 'us-east-2' + bucket, // 'bucket-name' + prefix, // 'my-prefix/' (optional) + key // 'data/stats.csv' (omits prefix, if any) +}); + +console.log(resp.headers); +``` + ### Return signed URL without fetching. ```js @@ -53,7 +105,7 @@ s3.sign({ prefix, key }); -``` +```` ### A note on S3 terminology diff --git a/bin/s3-download.js b/bin/s3-download.js index f0c5508..886c866 100755 --- a/bin/s3-download.js +++ b/bin/s3-download.js @@ -23,32 +23,34 @@ if (!key || !filepath) { async function run() { // GET STREAMED FILE - await s3 - .get({ - accessKeyId, - secretAccessKey, - region, - bucket, - prefix, - key - }) - .then(function (resp) { - console.log(resp.url); - return fs.promises.writeFile(filepath, resp.body); - }) - .catch(function (err) { - console.error('Error:'); - if (err.response) { - console.error(err.url); - console.error('GET Response:'); - console.error(err.response.statusCode); - console.error(err.response.headers); - console.error(err.response.body.toString('utf8')); - } else { - console.error(err); - } - process.exit(1); - }); + var resp = await s3.get({ + accessKeyId, + secretAccessKey, + region, + bucket, + prefix, + key, + stream: filepath + }); + + console.log('Downloading', resp.url); + await resp.stream; + + console.log(''); + console.log('Saved as', filepath); + console.log(''); } -run(); +run().catch(function (err) { + console.error('Error:'); + if (err.response) { + console.error(err.url); + console.error('GET Response:'); + console.error(err.response.statusCode); + console.error(err.response.headers); + console.error(err.response.body.toString('utf8')); + } else { + console.error(err); + } + process.exit(1); +}); diff --git a/index.js b/index.js index d556a4e..3d0839d 100644 --- a/index.js +++ b/index.js @@ -5,10 +5,40 @@ var request = require('@root/request'); var env = process.env; var S3; + +function toAwsBucketHost(host, bucket, region) { + if (host) { + return [host]; + } + + // Handle simply if it contains only valid subdomain characters + // (most notably that it does not have a '.' or '_') + if (/^[a-z0-9-]+$/i.test(bucket)) { + return ['', bucket + '.s3.amazonaws.com']; + } + + // Otherwise use region-specific handling rules + // (TODO: handle other regional exceptions) + // http://www.wryway.com/blog/aws-s3-url-styles/ + if (!region || 'us-east-1' === region) { + return ['s3.amazonaws.com']; + } + return ['s3-' + region + '.amazonaws.com']; +} + module.exports = S3 = { // HEAD head: function ( - { host, accessKeyId, secretAccessKey, region, bucket, prefix, key }, + { + host, + accessKeyId, + secretAccessKey, + region, + bucket, + prefix, + key, + ...requestOpts + }, _sign ) { // TODO support minio @@ -39,9 +69,10 @@ module.exports = S3 = { // whatever/ => whatever/ prefix = prefix.replace(/\/?$/, '/'); } + var [host, defaultHost] = toAwsBucketHost(host, bucket, region); var signed = aws4.sign( { - host: host || bucket + '.s3.amazonaws.com', + host: host || defaultHost, service: 's3', region: region, path: (host ? '/' + bucket : '') + '/' + prefix + key, @@ -55,7 +86,9 @@ module.exports = S3 = { return url; } - return request({ method: 'HEAD', url }).then(function (resp) { + return request( + Object.assign(requestOpts, { method: 'HEAD', url }) + ).then(function (resp) { if (200 === resp.statusCode) { resp.url = url; return resp; @@ -81,7 +114,8 @@ module.exports = S3 = { bucket, prefix, key, - json + json, + ...requestOpts }, _sign ) { @@ -89,9 +123,10 @@ module.exports = S3 = { if (prefix) { prefix = prefix.replace(/\/?$/, '/'); } + var [host, defaultHost] = toAwsBucketHost(host, bucket, region); var signed = aws4.sign( { - host: host || bucket + '.s3.amazonaws.com', + host: host || defaultHost, service: 's3', region: region, path: (host ? '/' + bucket : '') + '/' + prefix + key, @@ -110,12 +145,14 @@ module.exports = S3 = { if (json) { encoding = undefined; } - return request({ - method: 'GET', - url, - encoding: encoding, - json: json - }).then(function (resp) { + return request( + Object.assign(requestOpts, { + method: 'GET', + url, + encoding: encoding, + json: json + }) + ).then(function (resp) { if (200 === resp.statusCode) { resp.url = url; return resp; @@ -142,7 +179,8 @@ module.exports = S3 = { prefix, key, body, - size + size, + ...requestOpts }, _sign ) { @@ -150,9 +188,10 @@ module.exports = S3 = { if (prefix) { prefix = prefix.replace(/\/?$/, '/'); } + var [host, defaultHost] = toAwsBucketHost(host, bucket, region); var signed = aws4.sign( { - host: host || bucket + '.s3.amazonaws.com', + host: host || defaultHost, service: 's3', region: region, path: (host ? '/' + bucket : '') + '/' + prefix + key, @@ -167,9 +206,9 @@ module.exports = S3 = { headers['Content-Length'] = size; } - return request({ method: 'PUT', url, body, headers }).then(function ( - resp - ) { + return request( + Object.assign(requestOpts, { method: 'PUT', url, body, headers }) + ).then(function (resp) { if (200 === resp.statusCode) { resp.url = url; return resp; @@ -186,17 +225,27 @@ module.exports = S3 = { }, // DELETE - del: function ( - { host, accessKeyId, secretAccessKey, region, bucket, prefix, key }, + delete: function ( + { + host, + accessKeyId, + secretAccessKey, + region, + bucket, + prefix, + key, + ...requestOpts + }, _sign ) { prefix = prefix || ''; if (prefix) { prefix = prefix.replace(/\/?$/, '/'); } + var [host, defaultHost] = toAwsBucketHost(host, bucket, region); var signed = aws4.sign( { - host: host || bucket + '.s3.amazonaws.com', + host: host || defaultHost, service: 's3', region: region, path: (host ? '/' + bucket : '') + '/' + prefix + key, @@ -207,7 +256,9 @@ module.exports = S3 = { ); var url = 'https://' + signed.host + signed.path; - return request({ method: 'DELETE', url }).then(function (resp) { + return request( + Object.assign(requestOpts, { method: 'DELETE', url }) + ).then(function (resp) { if (204 === resp.statusCode) { resp.url = url; return resp; @@ -246,3 +297,4 @@ module.exports = S3 = { } } }; +S3.del = S3.delete; diff --git a/package-lock.json b/package-lock.json index e26c1b1..424258f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,13 +1,13 @@ { "name": "@root/s3", - "version": "1.1.3", + "version": "1.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { "@root/request": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@root/request/-/request-1.5.0.tgz", - "integrity": "sha512-J9RUIwVU99/cOVuDVYlNpr4G0A1/3ZxhCXIRiTZzu8RntOnb0lmDBMckhaus5ry9x/dBqJKDplFIgwHbLi6rLA==" + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@root/request/-/request-1.7.0.tgz", + "integrity": "sha512-lre7XVeEwszgyrayWWb/kRn5fuJfa+n0Nh+rflM9E+EpC28yIYA+FPm/OL1uhzp3TxhQM0HFN4FE2RDIPGlnmg==" }, "aws4": { "version": "1.9.1", diff --git a/package.json b/package.json index d38dcd9..7f57a50 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@root/s3", - "version": "1.1.3", + "version": "1.2.0", "description": "A simple, lightweight s3 client with only 2 dependencies", "main": "index.js", "bin": { @@ -13,6 +13,7 @@ "example": "examples" }, "scripts": { + "prettier": "npx prettier -w '**/*.js'", "test": "node test.js" }, "repository": { @@ -27,7 +28,7 @@ "author": "AJ ONeal (https://coolaj86.com/)", "license": "(MIT OR Apache-2.0)", "dependencies": { - "@root/request": "^1.5.0", + "@root/request": "^1.7.0", "aws4": "^1.9.1" }, "devDependencies": {