root
/
s3.js
mirror of https://github.com/therootcompany/s3.js
2
0
Fork 0
Code Issues Releases Wiki Activity

v1.2.0: bugfix for buckets with invalid domain names, and pass request options 

- Buckets with names like 'example.bucket' and 'example_bucket' are valid as paths, but not as domain names.
- allow passthrough support for latest @root/request, which supports pipes and streams
This commit is contained in:
AJ ONeal 2021-01-26 16:57:20 -07:00
parent ad0fa1f83b
commit efdfabb9a4
5 changed files with 163 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
README.md
View File

@ -13,8 +13,28 @@ A lightweight alternative to the S3 SDK that uses only @root/request and aws4.
### Download a file from S3
This library supports the same streaming options as [@root/request.js](https://git.rootprojects.org/root/request.js).
#### as a stream
```js
s3.get({
var resp = await s3.get({
accessKeyId, // 'AKIAXXXXXXXXXXXXXXXX'
secretAccessKey, // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
region, // 'us-east-2'
bucket, // 'bucket-name'
prefix, // 'my-prefix/' (optional)
key, // 'data/stats.csv' (omits prefix, if any)
stream // fs.createWriteStream('./path/to/file.bin')
});
await resp.stream;
```
#### in-memory
```js
var resp = await s3.get({
accessKeyId, // 'AKIAXXXXXXXXXXXXXXXX'
secretAccessKey, // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
region, // 'us-east-2'
@ -22,12 +42,14 @@ s3.get({
prefix, // 'my-prefix/' (optional)
key // 'data/stats.csv' (omits prefix, if any)
});
fs.writeFile(resp.body, './path/to/file.bin');
```
### Upload a new file to S3
```js
s3.set({
await s3.set({
accessKeyId,
secretAccessKey,
region,
@ -41,6 +63,36 @@ s3.set({
});
```
### Check that a file exists
```js
var resp = await s3.head({
accessKeyId, // 'AKIAXXXXXXXXXXXXXXXX'
secretAccessKey, // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
region, // 'us-east-2'
bucket, // 'bucket-name'
prefix, // 'my-prefix/' (optional)
key // 'data/stats.csv' (omits prefix, if any)
});
console.log(resp.headers);
```
### Delete file
```js
var resp = await s3.delete({
accessKeyId, // 'AKIAXXXXXXXXXXXXXXXX'
secretAccessKey, // 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
region, // 'us-east-2'
bucket, // 'bucket-name'
prefix, // 'my-prefix/' (optional)
key // 'data/stats.csv' (omits prefix, if any)
});
console.log(resp.headers);
```
### Return signed URL without fetching.
```js
@ -53,7 +105,7 @@ s3.sign({
prefix,
key
});
```
````
### A note on S3 terminology

56
bin/s3-download.js
View File

@ -23,32 +23,34 @@ if (!key || !filepath) {
async function run() {
// GET STREAMED FILE
await s3
.get({
accessKeyId,
secretAccessKey,
region,
bucket,
prefix,
key
})
.then(function (resp) {
console.log(resp.url);
return fs.promises.writeFile(filepath, resp.body);
})
.catch(function (err) {
console.error('Error:');
if (err.response) {
console.error(err.url);
console.error('GET Response:');
console.error(err.response.statusCode);
console.error(err.response.headers);
console.error(err.response.body.toString('utf8'));
} else {
console.error(err);
}
process.exit(1);
});
var resp = await s3.get({
accessKeyId,
secretAccessKey,
region,
bucket,
prefix,
key,
stream: filepath
});
console.log('Downloading', resp.url);
await resp.stream;
console.log('');
console.log('Saved as', filepath);
console.log('');
}
run();
run().catch(function (err) {
console.error('Error:');
if (err.response) {
console.error(err.url);
console.error('GET Response:');
console.error(err.response.statusCode);
console.error(err.response.headers);
console.error(err.response.body.toString('utf8'));
} else {
console.error(err);
}
process.exit(1);
});

92
index.js
View File

@ -5,10 +5,40 @@ var request = require('@root/request');
var env = process.env;
var S3;
function toAwsBucketHost(host, bucket, region) {
if (host) {
return [host];
}
// Handle simply if it contains only valid subdomain characters
// (most notably that it does not have a '.' or '_')
if (/^[a-z0-9-]+$/i.test(bucket)) {
return ['', bucket + '.s3.amazonaws.com'];
}
// Otherwise use region-specific handling rules
// (TODO: handle other regional exceptions)
// http://www.wryway.com/blog/aws-s3-url-styles/
if (!region || 'us-east-1' === region) {
return ['s3.amazonaws.com'];
}
return ['s3-' + region + '.amazonaws.com'];
}
module.exports = S3 = {
// HEAD
head: function (
{ host, accessKeyId, secretAccessKey, region, bucket, prefix, key },
{
host,
accessKeyId,
secretAccessKey,
region,
bucket,
prefix,
key,
...requestOpts
},
_sign
) {
// TODO support minio
@ -39,9 +69,10 @@ module.exports = S3 = {
// whatever/ => whatever/
prefix = prefix.replace(/\/?$/, '/');
}
var [host, defaultHost] = toAwsBucketHost(host, bucket, region);
var signed = aws4.sign(
{
host: host || bucket + '.s3.amazonaws.com',
host: host || defaultHost,
service: 's3',
region: region,
path: (host ? '/' + bucket : '') + '/' + prefix + key,
@ -55,7 +86,9 @@ module.exports = S3 = {
return url;
}
return request({ method: 'HEAD', url }).then(function (resp) {
return request(
Object.assign(requestOpts, { method: 'HEAD', url })
).then(function (resp) {
if (200 === resp.statusCode) {
resp.url = url;
return resp;
@ -81,7 +114,8 @@ module.exports = S3 = {
bucket,
prefix,
key,
json
json,
...requestOpts
},
_sign
) {
@ -89,9 +123,10 @@ module.exports = S3 = {
if (prefix) {
prefix = prefix.replace(/\/?$/, '/');
}
var [host, defaultHost] = toAwsBucketHost(host, bucket, region);
var signed = aws4.sign(
{
host: host || bucket + '.s3.amazonaws.com',
host: host || defaultHost,
service: 's3',
region: region,
path: (host ? '/' + bucket : '') + '/' + prefix + key,
@ -110,12 +145,14 @@ module.exports = S3 = {
if (json) {
encoding = undefined;
}
return request({
method: 'GET',
url,
encoding: encoding,
json: json
}).then(function (resp) {
return request(
Object.assign(requestOpts, {
method: 'GET',
url,
encoding: encoding,
json: json
})
).then(function (resp) {
if (200 === resp.statusCode) {
resp.url = url;
return resp;
@ -142,7 +179,8 @@ module.exports = S3 = {
prefix,
key,
body,
size
size,
...requestOpts
},
_sign
) {
@ -150,9 +188,10 @@ module.exports = S3 = {
if (prefix) {
prefix = prefix.replace(/\/?$/, '/');
}
var [host, defaultHost] = toAwsBucketHost(host, bucket, region);
var signed = aws4.sign(
{
host: host || bucket + '.s3.amazonaws.com',
host: host || defaultHost,
service: 's3',
region: region,
path: (host ? '/' + bucket : '') + '/' + prefix + key,
@ -167,9 +206,9 @@ module.exports = S3 = {
headers['Content-Length'] = size;
}
return request({ method: 'PUT', url, body, headers }).then(function (
resp
) {
return request(
Object.assign(requestOpts, { method: 'PUT', url, body, headers })
).then(function (resp) {
if (200 === resp.statusCode) {
resp.url = url;
return resp;
@ -186,17 +225,27 @@ module.exports = S3 = {
},
// DELETE
del: function (
{ host, accessKeyId, secretAccessKey, region, bucket, prefix, key },
delete: function (
{
host,
accessKeyId,
secretAccessKey,
region,
bucket,
prefix,
key,
...requestOpts
},
_sign
) {
prefix = prefix || '';
if (prefix) {
prefix = prefix.replace(/\/?$/, '/');
}
var [host, defaultHost] = toAwsBucketHost(host, bucket, region);
var signed = aws4.sign(
{
host: host || bucket + '.s3.amazonaws.com',
host: host || defaultHost,
service: 's3',
region: region,
path: (host ? '/' + bucket : '') + '/' + prefix + key,
@ -207,7 +256,9 @@ module.exports = S3 = {
);
var url = 'https://' + signed.host + signed.path;
return request({ method: 'DELETE', url }).then(function (resp) {
return request(
Object.assign(requestOpts, { method: 'DELETE', url })
).then(function (resp) {
if (204 === resp.statusCode) {
resp.url = url;
return resp;
@ -246,3 +297,4 @@ module.exports = S3 = {
}
}
};
S3.del = S3.delete;

8
package-lock.json generated
View File

@ -1,13 +1,13 @@
{
"name": "@root/s3",
"version": "1.1.3",
"version": "1.2.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@root/request": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/@root/request/-/request-1.5.0.tgz",
"integrity": "sha512-J9RUIwVU99/cOVuDVYlNpr4G0A1/3ZxhCXIRiTZzu8RntOnb0lmDBMckhaus5ry9x/dBqJKDplFIgwHbLi6rLA=="
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/@root/request/-/request-1.7.0.tgz",
"integrity": "sha512-lre7XVeEwszgyrayWWb/kRn5fuJfa+n0Nh+rflM9E+EpC28yIYA+FPm/OL1uhzp3TxhQM0HFN4FE2RDIPGlnmg=="
},
"aws4": {
"version": "1.9.1",

5
package.json
View File

@ -1,6 +1,6 @@
{
"name": "@root/s3",
"version": "1.1.3",
"version": "1.2.0",
"description": "A simple, lightweight s3 client with only 2 dependencies",
"main": "index.js",
"bin": {
@ -13,6 +13,7 @@
"example": "examples"
},
"scripts": {
"prettier": "npx prettier -w '**/*.js'",
"test": "node test.js"
},
"repository": {
@ -27,7 +28,7 @@
"author": "AJ ONeal <coolaj86@gmail.com> (https://coolaj86.com/)",
"license": "(MIT OR Apache-2.0)",
"dependencies": {
"@root/request": "^1.5.0",
"@root/request": "^1.7.0",
"aws4": "^1.9.1"
},
"devDependencies": {