2
0
mirror of https://github.com/therootcompany/request.js synced 2025-06-07 18:26:32 +00:00

Compare commits

...

65 Commits

Author SHA1 Message Date
bb30d5acf6
1.8.2 2022-06-14 01:37:28 -06:00
5b539deb7b
fix: copy http options from opts (not opts.uri) 2022-06-14 01:24:50 -06:00
0a2e7afa76
1.8.1 2022-01-12 12:45:07 -07:00
ba60df7eab
fix: don't JSON.stringify a string body, duh 2022-01-12 12:42:34 -07:00
bc4f6e59c0 1.8.0 2021-10-18 01:42:07 -06:00
3842ee1d61 docs: add CHANGELOG.md 2021-10-18 01:41:08 -06:00
9518cb970b feature: add resp.stream.body() and resp.ok 2021-10-18 01:38:44 -06:00
2e9a643c0f feature: fail faster on bad createWriteStream 2021-10-18 00:55:04 -06:00
dcd41a33d0 chore: add linter config 2021-10-18 00:47:39 -06:00
5f5e0b6066 bugfix: return after error callback 2021-10-18 00:47:06 -06:00
ed2bab931f
Update README.md 2021-08-20 22:57:43 -06:00
a95d003ed5
docs: link to form-data v2.x (JavaScript) docs 2021-08-06 15:27:06 -06:00
5149bc9dcb
docs: clarify stream usage 2021-08-06 14:07:03 -06:00
95a12a8285 v1.7.0 add stream support 2021-01-14 16:35:07 -07:00
9395ec96e3 make Prettier (v2) 2021-01-14 16:07:48 -07:00
3574e35635 v1.6.1: userAgent and docs 2020-04-28 23:14:50 -06:00
508f1ce591 add comments 2020-04-28 23:00:26 -06:00
c2c4b5b2de v1.6.0: add default User-Agent, bugfixes 2020-04-28 22:58:11 -06:00
f6557b7610 handle possible readable body error 2020-04-28 22:56:12 -06:00
4b9a1f07ee add default User-Agent 2020-04-28 22:43:27 -06:00
e22baa8eae v1.5.0: add pipe support, fix bearer bug 2020-03-12 02:01:23 -06:00
812f4e6062 bugfix request compat opts.auth.bearer 2020-03-12 01:59:33 -06:00
5b5cd36aa5 allow pipe-able inputs 2020-03-12 01:57:10 -06:00
4f3fe38ee4 update docs for promises 2020-02-27 02:17:07 +00:00
ef3183e984 add 80/20 message 2020-02-20 17:34:27 +00:00
e384aead9b v1.4.2: regression fix: copy 'method' to final object 2019-11-01 01:19:47 -06:00
87bf5a5fc5 v1.4.2: regression fix: copy 'method' to final object 2019-11-01 01:18:25 -06:00
1692b9e7df v1.4.1: GET by default *after* POST body check 2019-10-29 21:53:39 -06:00
8f5133385b v1.4.0: add promise support and easy debugging 2019-10-29 14:57:28 -06:00
1507b38503 v1.3.12: bugfix json request 2019-10-29 14:42:26 -06:00
9ab91d9721 make Prettier 2019-10-29 14:31:30 -06:00
b6900b937b remove extraneous variable 2019-06-04 04:20:29 +00:00
be91c1b78f show promise usage 2019-06-04 04:19:11 +00:00
335dd98631 v1.3.11: pass callback 2019-05-23 13:25:20 -06:00
28f3f783ae v1.3.10: update header and link 2019-04-16 12:43:21 -06:00
1d4c3c1997 v1.3.9: update name and urls 2019-04-16 12:41:05 -06:00
999797270f v1.3.8: A Root Project 2019-04-04 01:39:16 -06:00
AJ ONeal
aec78877cf v1.3.7: agent passthru (also timeout, setHost, createConnection, family, host, localAddress) 2019-02-26 01:02:37 -07:00
4eaf68966c note the need to install form-data@2 when using forms 2018-09-16 21:02:57 -06:00
49daa68225 v1.3.6: merge some bugfix PRs 2018-07-12 03:12:49 -06:00
AJ ONeal
c880788e9a Merge branch 'fix/getHeaderName' of Joxit/urequest.js into master 2018-07-12 09:05:15 +00:00
AJ ONeal
c4ec14fb6a Merge branch 'redirect' of Joxit/urequest.js into master 2018-07-12 09:01:02 +00:00
Joxit
b3f4042adb Should send opts and not opts.headers in getHeaderName 2018-07-12 10:17:59 +02:00
Joxit
e1bc24ce97 Add HTTP codes 307 and 308 for redirect
See https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
2018-07-12 10:10:23 +02:00
137da9b903 v1.3.5 2018-07-07 19:57:16 -06:00
078f922635 stop on form-data error 2018-07-07 19:56:53 -06:00
3defd84af4 v1.3.4: bugfix keep separate headers and don't keep Content-Length on redirect 2018-07-07 19:46:21 -06:00
50704fdccc v1.3.3: bugfix form-data request object 2018-07-07 16:19:00 -06:00
7255134e9f v1.3.2: bugfix form-data 2018-07-07 14:31:36 -06:00
739354090b wrap onResponse for form-data's signature with error 2018-07-07 14:30:58 -06:00
c355fb20c4 explicit POST and comment on possible deviation from request API 2018-07-06 22:40:28 -06:00
aa509ea4ce v1.3.1: bugfix regression with json on GET 2018-07-06 22:32:28 -06:00
5150e076cb bugfix: don't default to POST when json: true 2018-07-06 22:31:37 -06:00
f81dce9786 v1.3.0: support Basic and Bearer auth 2018-07-06 21:24:23 -06:00
e0fe41838d support Basic and Bearer auth 2018-07-06 21:23:46 -06:00
7683d35958 request => urequest 2018-07-06 20:41:35 -06:00
2d95ed2ac0 v1.2.1: bump npm latest 2018-07-06 20:28:53 -06:00
ba76c3aa14 v1.2.0: add form and formData support 2018-07-06 20:25:46 -06:00
b5714c6a65 www-form support, add *Content-Length by default 2018-07-06 20:20:14 -06:00
8758964610 FormData support with form-data 2018-07-06 19:38:47 -06:00
d5af2ae5fe v1.1.4 2018-06-29 02:20:33 -06:00
db01707e13 return correct obj and ws 2018-06-29 02:20:07 -06:00
dcc5ba0c7e add unix domain socket support and update README 2018-06-29 02:18:44 -06:00
2be278364c v1.1.3 2018-06-26 14:44:19 -06:00
94eec37646 use callback not throw 2018-06-26 14:43:56 -06:00
20 changed files with 1306 additions and 334 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
node_modules
.*.sw*

22
.jshintrc Normal file
View File

@ -0,0 +1,22 @@
{
"browser": true,
"node": true,
"esversion": 11,
"curly": true,
"sub": true,
"bitwise": true,
"eqeqeq": true,
"forin": true,
"freeze": true,
"immed": true,
"latedef": "nofunc",
"nonbsp": true,
"nonew": true,
"plusplus": true,
"undef": true,
"unused": "vars",
"strict": true,
"maxdepth": 3,
"maxstatements": 100,
"maxcomplexity": 40
}

8
.prettierrc Normal file
View File

@ -0,0 +1,8 @@
{
"bracketSpacing": true,
"printWidth": 80,
"singleQuote": true,
"tabWidth": 4,
"trailingComma": "none",
"useTabs": false
}

6
CHANGELOG.md Normal file
View File

@ -0,0 +1,6 @@
# CHANGELOG
## v1.8.0
- add `resp.ok` - same as WHATWG fetch `resp.ok = (resp.statusCode >= 200 && resp.statusCode < 300)`
- add `resp.stream.body()` to populate `resp.body` rather than (or perhaps in addition to) continuing to stream (useful for error handling)

65
EXTRA.md Normal file
View File

@ -0,0 +1,65 @@
# Extra
There are some niche features of @root/request which are beyond the request.js
compatibility.
## async/await & Promises
The differences in async support are explained in [README.md](/README.md), up near the top.
If you're familiar with Promises (and async/await), then it's pretty self-explanatory.
## ok
Just like WHATWG `fetch`, we have `resp.ok`:
```js
let resp = await request({
url: 'https://example.com'
}).then(mustOk);
```
```js
function mustOk(resp) {
if (!resp.ok) {
// handle error
throw new Error('BAD RESPONSE');
}
return resp;
}
```
## streams
The differences in stream support are explained in [README.md](/README.md), up near the top.
## userAgent
There's a default User-Agent string describing the version of @root/request, node.js, and the OS.
Add to the default User-Agent
```js
request({
// ...
userAgent: 'my-package/1.0' // add to agent
});
```
Replace the default User-Agent
```js
request({
// ...
headers: { 'User-Agent': 'replace/0.0' }
});
```
Disable the default User-Agent:
```js
request({
// ...
headers: { 'User-Agent': false }
});
```

366
README.md
View File

@ -1,19 +1,25 @@
# µRequest - Minimalist HTTP client
# [@root/request](https://git.rootprojects.org/root/request.js) | a [Root](https://rootprojects.org) project
A lightweight alternative to (and drop-in replacement for) request.
> Minimalist HTTP client
Written from scratch.
A lightweight alternative to (and 80/20 drop-in replacement for) request.
Has the 20% of features that 80%+ of people need, in about 500 LoC.
Written from scratch, with zero-dependencies.
## Super simple to use
µRequest is designed to be a drop-in replacement for request. It supports HTTPS and follows redirects by default.
@root/request is designed to be a drop-in replacement for request. It also supports Promises and async/await by default, enhanced stream support, and a few other things as mentioned below.
```bash
npm install --save @coolaj86/urequest
npm install --save @root/request
# or npm install git+ssh://git@git.therootcompany.com/request.js
```
```js
var request = require('@coolaj86/urequest');
var request = require('@root/request');
request('http://www.google.com', function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
@ -21,6 +27,327 @@ request('http://www.google.com', function (error, response, body) {
});
```
**Using Promises**
```js
var request = require('@root/request');
request('http://www.google.com')
.then(function (response) {
console.log('statusCode:', response.statusCode); // Print the response status code if a response was received
console.log('body:', response.body); // Print the HTML for the Google homepage.
})
.catch(function (error) {
console.log('error:', error); // Print the error if one occurred
});
```
**Streaming**
In order to keep this library lightweight, performant, and keep the code easy to
read, the streaming behavior is **_slightly different_** from that of
`request.js`.
```diff
-var request = require('request');
+var request = require('@root/request');
-var stream = request({ url, headers });
+var stream = await request({ url, headers });
let attachment = await new MailgunAPI.Attachment({
data: stream
})
```
Example:
```js
var request = require('@root/request');
var resp = await request({
url: 'http://www.google.com',
stream: true // true | 'filename.ext' | stream.Writable
});
// 'resp' itself is a ReadableStream
resp.on('data', function () {
// got some data
});
resp.on('end', function () {
// the data has ended
});
// 'resp.stream' is a Promise that is resolved when the read stream is destroyed
await resp.stream; // returns `undefined`
console.log('Done');
```
The difference is that we don't add an extra layer of stream abstraction.
You must use the response from await, a Promise, or the callback.
You can also give a file path:
```js
request({
url: 'http://www.google.com',
stream: '/tmp/google-index.html'
});
```
Which is equivalent to passing a write stream for the file:
```js
request({
url: 'http://www.google.com',
stream: fs.createWriteStream('/tmp/google-index.html')
});
```
Also, `await resp.stream.body()` can be used to get back the full body (the same as if you didn't use the `stream` option:
```js
let resp = await request({
url: 'http://www.google.com',
stream: true
});
if (!resp.ok) {
await resp.stream.body();
console.error(resp.body);
}
```
## Table of contents
- [Extra Features](/EXTRA.md)
- [Forms](#forms)
- [HTTP Authentication](#http-authentication)
- [Custom HTTP Headers](#custom-http-headers)
- [Unix Domain Sockets](#unix-domain-sockets)
- [**All Available Options**](#requestoptions-callback)
## Extra Features
The following are features that the original `request` did not have, but have been added for convenience in `@root/request`.
- Support for `async`/`await` & `Promise`s (as explained above)
- `request({ userAgent: 'my-api/1.1' })` (for building API clients)
- `resp.ok` (just like `fetch`)
- `resp.stream` (see above)
See [EXTRA.md](/EXTRA.md)
## Forms
`@root/request` supports `application/x-www-form-urlencoded` and `multipart/form-data` form uploads.
<!-- For `multipart/related` refer to the `multipart` API. -->
#### application/x-www-form-urlencoded (URL-Encoded Forms)
URL-encoded forms are simple.
```js
request.post('http://service.com/upload', { form: { key: 'value' } });
// or
request.post(
{ url: 'http://service.com/upload', form: { key: 'value' } },
function (err, httpResponse, body) {
/* ... */
}
);
```
<!--
// or
request.post('http://service.com/upload').form({key:'value'})
-->
#### multipart/form-data (Multipart Form Uploads)
For `multipart/form-data` we use the [form-data](https://github.com/form-data/form-data/tree/v2.5.1) library by [@felixge](https://github.com/felixge). For the most cases, you can pass your upload form data via the `formData` option.
To use `form-data`, you must install it separately:
```bash
npm install --save form-data@2.x
```
```js
var formData = {
// Pass a simple key-value pair
my_field: 'my_value',
// Pass data via Buffers
my_buffer: Buffer.from([1, 2, 3]),
// Pass data via Streams
my_file: fs.createReadStream(__dirname + '/unicycle.jpg'),
// Pass multiple values /w an Array
attachments: [
fs.createReadStream(__dirname + '/attachment1.jpg'),
fs.createReadStream(__dirname + '/attachment2.jpg')
],
// Pass optional meta-data with an 'options' object with style: {value: DATA, options: OPTIONS}
// Use case: for some types of streams, you'll need to provide "file"-related information manually.
// See the `form-data` README for more information about options: https://github.com/form-data/form-data
custom_file: {
value: fs.createReadStream('/dev/urandom'),
options: {
filename: 'topsecret.jpg',
contentType: 'image/jpeg'
}
}
};
request.post(
{ url: 'http://service.com/upload', formData: formData },
function optionalCallback(err, httpResponse, body) {
if (err) {
return console.error('upload failed:', err);
}
console.log('Upload successful! Server responded with:', body);
}
);
```
<!--
For advanced cases, you can access the form-data object itself via `r.form()`. This can be modified until the request is fired on the next cycle of the event-loop. (Note that this calling `form()` will clear the currently set form data for that request.)
```js
// NOTE: Advanced use-case, for normal use see 'formData' usage above
var r = request.post('http://service.com/upload', function optionalCallback(err, httpResponse, body) {...})
var form = r.form();
form.append('my_field', 'my_value');
form.append('my_buffer', Buffer.from([1, 2, 3]));
form.append('custom_file', fs.createReadStream(__dirname + '/unicycle.jpg'), {filename: 'unicycle.jpg'});
```
-->
See the [form-data README](https://github.com/form-data/form-data) for more information & examples.
---
## HTTP Authentication
<!--
request.get('http://some.server.com/').auth('username', 'password', false);
// or
request.get('http://some.server.com/').auth(null, null, true, 'bearerToken');
// or
-->
```js
request.get('http://some.server.com/', {
auth: {
user: 'username',
pass: 'password',
sendImmediately: false
}
});
// or
request.get('http://some.server.com/', {
auth: {
bearer: 'bearerToken'
}
});
```
If passed as an option, `auth` should be a hash containing values:
- `user` || `username`
- `pass` || `password`
- `bearer` (optional)
<!--
- `sendImmediately` (optional)
The method form takes parameters
`auth(username, password, sendImmediately, bearer)`.
`sendImmediately` defaults to `true`, which causes a basic or bearer
authentication header to be sent. If `sendImmediately` is `false`, then
`request` will retry with a proper authentication header after receiving a
`401` response from the server (which must contain a `WWW-Authenticate` header
indicating the required authentication method).
-->
Note that you can also specify basic authentication using the URL itself, as
detailed in [RFC 1738](http://www.ietf.org/rfc/rfc1738.txt). Simply pass the
`user:password` before the host with an `@` sign:
```js
var username = 'username',
password = 'password',
url = 'http://' + username + ':' + password + '@some.server.com';
request({ url: url }, function (error, response, body) {
// Do more stuff with 'body' here
});
```
<!--
Digest authentication is supported, but it only works with `sendImmediately`
set to `false`; otherwise `request` will send basic authentication on the
initial request, which will probably cause the request to fail.
-->
Bearer authentication is supported, and is activated when the `bearer` value is
available. The value may be either a `String` or a `Function` returning a
`String`. Using a function to supply the bearer token is particularly useful if
used in conjunction with `defaults` to allow a single function to supply the
last known token at the time of sending a request, or to compute one on the fly.
[back to top](#table-of-contents)
---
## Custom HTTP Headers
HTTP Headers, such as `User-Agent`, can be set in the `options` object.
In the example below, we call the github API to find out the number
of stars and forks for the request repository. This requires a
custom `User-Agent` header as well as https.
```js
var request = require('request');
var options = {
url: 'https://api.github.com/repos/request/request',
headers: {
'User-Agent': 'request'
}
};
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
var info = JSON.parse(body);
console.log(info.stargazers_count + ' Stars');
console.log(info.forks_count + ' Forks');
}
}
request(options, callback);
```
[back to top](#table-of-contents)
---
## UNIX Domain Sockets
`@root/request` supports making requests to [UNIX Domain Sockets](https://en.wikipedia.org/wiki/Unix_domain_socket). To make one, use the following URL scheme:
```js
/* Pattern */ 'http://unix:SOCKET:PATH';
/* Example */ request.get(
'http://unix:/absolute/path/to/unix.socket:/request/path'
);
```
Note: The `SOCKET` path is assumed to be absolute to the root of the host file system.
[back to top](#table-of-contents)
---
## request(options, callback)
@ -92,39 +419,42 @@ instead, it **returns a wrapper** that has your default settings applied to it.
`request.defaults` to add/override defaults that were previously defaulted.
For example:
```js
//requests using baseRequest() will set the 'x-token' header
var baseRequest = request.defaults({
headers: { 'x-token': 'my-token' }
})
});
//requests using specialRequest() will include the 'x-token' header set in
//baseRequest and will also include the 'special' header
var specialRequest = baseRequest.defaults({
headers: { special: 'special value' }
})
});
```
### request.METHOD()
These HTTP method convenience functions act just like `request()` but with a default method already set for you:
- *request.get()*: Defaults to `method: "GET"`.
- *request.post()*: Defaults to `method: "POST"`.
- *request.put()*: Defaults to `method: "PUT"`.
- *request.patch()*: Defaults to `method: "PATCH"`.
- *request.del() / request.delete()*: Defaults to `method: "DELETE"`.
- *request.head()*: Defaults to `method: "HEAD"`.
- *request.options()*: Defaults to `method: "OPTIONS"`.
- _request.get()_: Defaults to `method: "GET"`.
- _request.post()_: Defaults to `method: "POST"`.
- _request.put()_: Defaults to `method: "PUT"`.
- _request.patch()_: Defaults to `method: "PATCH"`.
- _request.del() / request.delete()_: Defaults to `method: "DELETE"`.
- _request.head()_: Defaults to `method: "HEAD"`.
- _request.options()_: Defaults to `method: "OPTIONS"`.
---
## Debugging
There are at least <!--three--> two ways to debug the operation of `request`:
1. Launch the node process like `NODE_DEBUG=urequest node script.js`
1. Launch the node process like `NODE_DEBUG=@root/request node script.js`
(`lib,request,otherlib` works too).
2. Set `require('@coolaj86/urequest').debug = true` at any time (this does the same thing
2. Set `require('@root/request').debug = true` at any time (this does the same thing
as #1).
<!-- TODO
@ -133,3 +463,5 @@ There are at least <!--three--> two ways to debug the operation of `request`:
[back to top](#table-of-contents)
-->
[back to top](#table-of-contents)

View File

@ -5,7 +5,9 @@ var request = require('../');
// will redirect to https://www.github.com and then https://github.com
//request('http://www.github.com', function (error, response, body) {
request({ uri: { protocol: 'http:', hostname: 'www.github.com' } }, function (error, response, body) {
request(
{ uri: { protocol: 'http:', hostname: 'www.github.com' } },
function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;
@ -13,4 +15,5 @@ request({ uri: { protocol: 'http:', hostname: 'www.github.com' } }, function (er
console.log('statusCode:', response.statusCode); // The final statusCode
console.log('Final href:', response.request.uri.href); // The final URI
console.log('Body Length:', body.length); // body length
});
}
);

31
examples/form-data.js Normal file
View File

@ -0,0 +1,31 @@
'use strict';
//var request = require('@coolaj86/urequest');
// To check and make sure the outputs are the same
//var request = require('request');
var request = require('../');
// will redirect to https://www.github.com and then https://github.com
//request('http://www.github.com', function (error, response, body) {
request(
//{ url: 'http://postb.in/syfxxnko'
{
url: 'http://localhost:3007/form-data/',
method: 'POST',
headers: { 'X-Foo': 'Bar' },
formData: {
foo: 'bar',
baz: require('fs').createReadStream(
require('path').join(__dirname, 'get-to-json.js')
)
}
},
function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;
}
console.log('statusCode:', response.statusCode); // The final statusCode
console.log('Body Length:', body.length); // body length
}
);

View File

@ -4,7 +4,9 @@
var request = require('../');
// would normally redirect to https://www.github.com and then https://github.com
request({ uri: 'https://www.github.com', followRedirect: false }, function (error, response, body) {
request(
{ uri: 'https://www.github.com', followRedirect: false },
function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;
@ -13,4 +15,5 @@ request({ uri: 'https://www.github.com', followRedirect: false }, function (erro
console.log('statusCode:', response.statusCode); // Should be 301 or 302
console.log('Location:', response.headers.location); // The redirect
console.log('Body:', body || JSON.stringify(body));
});
}
);

15
examples/postbin.js Normal file
View File

@ -0,0 +1,15 @@
'use strict';
var request = require('../');
request({
url: 'https://postb.in/1588134650162-6019286897499?hello=world'
//headers: { 'user-agent': false } // remove
//headers: { 'user-agent': 'test/1.0' } // overwrite
//userAgent: 'test/1.1' // add to the default
})
.then(function (resp) {
console.log(resp.body);
})
.catch(function (err) {
console.error(err);
});

View File

@ -0,0 +1,27 @@
'use strict';
var request = require('../');
async function main() {
var tpath = '/tmp/google-index.html';
var resp = await request({
url: 'https://google.com',
encoding: null,
stream: tpath
});
console.log('[Response Headers]');
console.log(resp.toJSON().headers);
//console.error(resp.headers, resp.body.byteLength);
await resp.stream;
console.log('[Response Body] written to', tpath);
}
main()
.then(function () {
console.log('Pass');
})
.catch(function (e) {
console.error('Fail');
console.error(e.stack);
});

34
examples/stream.js Normal file
View File

@ -0,0 +1,34 @@
'use strict';
var request = require('../');
async function main() {
var tpath = '/tmp/google-index.html';
var resp = await request({
url: 'https://google.com',
encoding: null,
stream: true
});
console.log('[Response Headers]');
console.log(resp.toJSON().headers);
resp.on('data', function (chunk) {
console.log('[Data]', chunk.byteLength);
});
resp.on('end', function (chunk) {
console.log('[End]');
});
//console.error(resp.headers, resp.body.byteLength);
await resp.stream;
console.log('[Close]');
}
main()
.then(function () {
console.log('Pass');
})
.catch(function (e) {
console.error('Fail');
console.error(e.stack);
});

26
examples/www-form.js Normal file
View File

@ -0,0 +1,26 @@
'use strict';
//var request = require('@coolaj86/urequest');
// To check and make sure the outputs are the same
//var request = require('request');
var request = require('../');
// will redirect to https://www.github.com and then https://github.com
//request('http://www.github.com', function (error, response, body) {
request(
{
url: 'http://postb.in/2meyt50C',
method: 'POST',
headers: { 'X-Foo': 'Bar' },
form: { foo: 'bar', baz: 'qux' }
},
function (error, response, body) {
if (error) {
console.log('error:', error); // Print the error if one occurred
return;
}
console.log('statusCode:', response.statusCode); // The final statusCode
console.log('Body Length:', body.length); // body length
console.log('Response:', response.toJSON()); // body length
}
);

579
index.js
View File

@ -3,6 +3,9 @@
var http = require('http');
var https = require('https');
var url = require('url');
var os = require('os');
var pkg = require('./package.json');
var fs = require('fs'); // only for streams
function debug() {
if (module.exports.debug) {
@ -21,7 +24,10 @@ function mergeOrDelete(defaults, updates) {
// CRDT probs...
if ('undefined' === typeof updates[key]) {
delete updates[key];
} else if ('object' === typeof defaults[key] && 'object' === typeof updates[key]) {
} else if (
'object' === typeof defaults[key] &&
'object' === typeof updates[key]
) {
updates[key] = mergeOrDelete(defaults[key], updates[key]);
}
});
@ -29,16 +35,21 @@ function mergeOrDelete(defaults, updates) {
return updates;
}
function hasHeader(reqOpts, header) {
var headers = {};
Object.keys(reqOpts.headers).forEach(function (key) {
headers[key.toLowerCase()] = true;
// retrieves an existing header, case-sensitive
function getHeaderName(reqOpts, header) {
var headerNames = {};
Object.keys(reqOpts.headers).forEach(function (casedName) {
headerNames[casedName.toLowerCase()] = casedName;
});
return headers[header.toLowerCase()];
// returns the key, which in erroneous cases could be an empty string
return headerNames[header.toLowerCase()];
}
// returns whether or not a header exists, case-insensitive
function hasHeader(reqOpts, header) {
return 'undefined' !== typeof getHeaderName(reqOpts, header);
}
function toJSONifier(keys) {
return function () {
var obj = {};
var me = this;
@ -55,73 +66,72 @@ function toJSONifier(keys) {
};
}
function setDefaults(defs) {
defs = defs || {};
function urequestHelper(opts, cb) {
debug("\n[urequest] processed options:");
debug(opts);
function onResponse(resp) {
var followRedirect;
Object.keys(defs).forEach(function (key) {
if (key in opts && 'undefined' !== typeof opts[key]) {
return;
}
opts[key] = defs[key];
function setupPipe(resp, opts) {
// make the response await-able
var resolve;
var reject;
var p = new Promise(function (_resolve, _reject) {
resolve = _resolve;
reject = _reject;
});
followRedirect = opts.followRedirect;
resp.toJSON = toJSONifier([ 'statusCode', 'body', 'headers', 'request' ]);
// or an existing write stream
if ('function' === typeof opts.stream.pipe) {
if (opts.debug) {
console.debug('[@root/request] stream piped');
}
resp.pipe(opts.stream);
}
resp.once('error', function (e) {
if (opts.debug) {
console.debug("[@root/request] stream 'error'");
console.error(e.stack);
}
resp.destroy();
if ('function' === opts.stream.destroy) {
opts.stream.destroy(e);
}
reject(e);
});
resp.once('end', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'end'");
}
if ('function' === opts.stream.destroy) {
opts.stream.end();
// this will close the stream (i.e. sync to disk)
opts.stream.destroy();
}
});
resp.once('close', function () {
if (opts.debug) {
console.debug("[@root/request] stream 'close'");
}
resolve();
});
return p;
}
resp.request = req;
resp.request.uri = url.parse(opts.url);
//resp.request.method = opts.method;
resp.request.headers = opts.headers;
resp.request.toJSON = toJSONifier([ 'uri', 'method', 'headers' ]);
if (followRedirect && resp.headers.location && -1 !== [ 301, 302 ].indexOf(resp.statusCode)) {
debug('Following redirect: ' + resp.headers.location);
if ('GET' !== opts.method && !opts.followAllRedirects) {
followRedirect = false;
}
if (opts._redirectCount >= opts.maxRedirects) {
followRedirect = false;
}
if ('function' === opts.followRedirect) {
if (!opts.followRedirect(resp)) {
followRedirect = false;
}
}
if (followRedirect) {
if (!opts.followOriginalHttpMethod) {
opts.method = 'GET';
opts.body = null;
}
if (opts.removeRefererHeader && opts.headers) {
delete opts.headers.referer;
}
opts.url = resp.headers.location;
opts.uri = url.parse(opts.url);
return urequestHelper(opts, cb);
}
}
function handleResponse(resp, opts, cb) {
// body can be buffer, string, or json
if (null === opts.encoding) {
resp._body = [];
} else {
resp.body = '';
}
resp._bodyLength = 0;
resp.on('data', function (chunk) {
resp.on('readable', function () {
var chunk;
while ((chunk = resp.read())) {
if ('string' === typeof resp.body) {
resp.body += chunk.toString(opts.encoding);
} else {
resp._body.push(chunk);
resp._bodyLength += chunk.length;
}
}
});
resp.on('end', function () {
resp.once('end', function () {
if ('string' !== typeof resp.body) {
if (1 === resp._body.length) {
resp.body = resp._body[0];
@ -140,24 +150,159 @@ function setDefaults(defs) {
}
}
debug("\n[urequest] resp.toJSON():");
debug('\n[urequest] resp.toJSON():');
if (module.exports.debug) {
debug(resp.toJSON());
}
if (opts.debug) {
console.debug('[@root/request] Response Body:');
console.debug(resp.body);
}
cb(null, resp, resp.body);
});
}
function setDefaults(defs) {
defs = defs || {};
function urequestHelper(opts, cb) {
debug('\n[urequest] processed options:');
debug(opts);
var req;
var finalOpts = {};
// allow specifying a file
if ('string' === typeof opts.stream) {
if (opts.debug) {
console.debug('[@root/request] creating file write stream');
}
try {
opts.stream = fs.createWriteStream(opts.stream);
} catch (e) {
cb(e);
return;
}
}
function onResponse(resp) {
var followRedirect;
Object.keys(defs).forEach(function (key) {
if (key in opts && 'undefined' !== typeof opts[key]) {
return;
}
opts[key] = defs[key];
});
followRedirect = opts.followRedirect;
// copied from WHATWG fetch
resp.ok = false;
if (resp.statusCode >= 200 && resp.statusCode < 300) {
resp.ok = true;
}
resp.toJSON = toJSONifier([
'statusCode',
'body',
'ok',
'headers',
'request'
]);
resp.request = req;
resp.request.uri = url.parse(opts.url);
//resp.request.method = opts.method;
resp.request.headers = finalOpts.headers;
resp.request.toJSON = toJSONifier(['uri', 'method', 'headers']);
if (opts.debug) {
console.debug('[@root/request] Response Headers:');
console.debug(resp.toJSON());
}
if (
followRedirect &&
resp.headers.location &&
-1 !== [301, 302, 307, 308].indexOf(resp.statusCode)
) {
debug('Following redirect: ' + resp.headers.location);
if ('GET' !== opts.method && !opts.followAllRedirects) {
followRedirect = false;
}
if (opts._redirectCount >= opts.maxRedirects) {
followRedirect = false;
}
if ('function' === opts.followRedirect) {
if (!opts.followRedirect(resp)) {
followRedirect = false;
}
}
if (followRedirect) {
if (!opts.followOriginalHttpMethod) {
opts.method = 'GET';
opts.body = null;
delete opts.headers[
getHeaderName(opts, 'Content-Length')
];
delete opts.headers[
getHeaderName(opts, 'Transfer-Encoding')
];
}
if (opts.removeRefererHeader && opts.headers) {
delete opts.headers.referer;
}
// TODO needs baseUrl, maybe test for host / socketPath?
opts.url = resp.headers.location;
opts.uri = url.parse(opts.url);
return urequestHelper(opts, cb);
}
}
if (opts.stream) {
resp.stream = setupPipe(resp, opts);
// can be string, buffer, or json... why not an async function too?
resp.stream.body = async function () {
handleResponse(resp, opts, cb);
await resp.stream;
return resp.body;
};
cb(null, resp);
return;
}
handleResponse(resp, opts, cb);
}
var _body;
var MyFormData;
var form;
var formHeaders;
var requester;
if (opts.body) {
if (true === opts.json) {
if (true === opts.json && 'string' !== typeof opts.body) {
_body = JSON.stringify(opts.body);
} else {
_body = opts.body;
}
} else if (opts.json && true !== opts.json) {
_body = JSON.stringify(opts.json);
} else if (opts.form) {
_body = Object.keys(opts.form)
.filter(function (key) {
if ('undefined' !== typeof opts.form[key]) {
return true;
}
})
.map(function (key) {
return (
encodeURIComponent(key) +
'=' +
encodeURIComponent(String(opts.form[key]))
);
})
.join('&');
opts.headers['Content-Type'] = 'application/x-www-form-urlencoded';
}
if ('string' === typeof _body) {
_body = Buffer.from(_body);
@ -166,47 +311,203 @@ function setDefaults(defs) {
Object.keys(opts.uri).forEach(function (key) {
finalOpts[key] = opts.uri[key];
});
// A bug should be raised if request does it differently,
// but I think we're supposed to pass all acceptable options
// on to the raw http request
[
'family',
'host',
'localAddress',
'agent',
'createConnection',
'timeout',
'setHost'
].forEach(function (key) {
if (key in opts) {
finalOpts[key] = opts[key];
}
});
finalOpts.method = opts.method;
finalOpts.headers = opts.headers;
finalOpts.headers = JSON.parse(JSON.stringify(opts.headers));
var uaHeader = getHeaderName(finalOpts, 'User-Agent') || 'User-Agent';
// set a default user-agent
if (!finalOpts.headers[uaHeader]) {
if (false === finalOpts.headers[uaHeader]) {
delete finalOpts.headers[uaHeader];
} else {
finalOpts.headers[uaHeader] = getUserAgent(opts.userAgent);
}
}
if (_body) {
// Most APIs expect (or require) Content-Length except in the case of multipart uploads
// chunked is generally only well-supported downstream
//finalOpts.headers['Content-Length'] = _body.byteLength || _body.length;
// Transfer-Encoding: Chunked (the default) is generally only well-supported downstream
if (
'undefined' !== typeof _body.byteLength ||
'undefined' !== typeof _body.length
) {
finalOpts.headers['Content-Length'] =
_body.byteLength || _body.length;
}
}
if (opts.auth) {
// if opts.uri specifies auth it will be parsed by url.parse and passed directly to the http module
if ('string' !== typeof opts.auth) {
opts.auth =
(opts.auth.user || opts.auth.username || '') +
':' +
(opts.auth.pass || opts.auth.password || '');
}
if ('string' === typeof opts.auth) {
finalOpts.auth = opts.auth;
}
if (false === opts.sendImmediately) {
console.warn(
'[Warn] setting `sendImmediately: false` is not yet supported. Please open an issue if this is an important feature that you need.'
);
}
// [request-compat]
if (opts.auth.bearer) {
// having a shortcut for base64 encoding makes sense, but this? Eh, whatevs...
finalOpts.header.Authorization = 'Bearer ' + opts.auth.bearer;
}
}
if (opts.formData) {
try {
MyFormData = opts.FormData || require('form-data');
// potential options https://github.com/felixge/node-combined-stream/blob/master/lib/combined_stream.js#L7-L15
} catch (e) {
console.error(
'urequest does not include extra dependencies by default'
);
console.error(
"if you need to use 'form-data' you may install it, like so:"
);
console.error(' npm install --save form-data');
cb(e);
return;
}
try {
form = new MyFormData();
Object.keys(opts.formData).forEach(function (key) {
function add(key, data, opts) {
if (data.value) {
opts = data.options;
data = data.value;
}
form.append(key, data, opts);
}
if (Array.isArray(opts.formData[key])) {
opts.formData[key].forEach(function (data) {
add(key, data);
});
} else {
add(key, opts.formData[key]);
}
});
} catch (e) {
cb(e);
return;
}
formHeaders = form.getHeaders();
Object.keys(formHeaders).forEach(function (header) {
finalOpts.headers[header] = formHeaders[header];
});
}
// TODO support unix sockets
if ('https:' === finalOpts.protocol) {
// https://nodejs.org/api/https.html#https_https_request_options_callback
debug("\n[urequest] https.request(opts):");
debug('\n[urequest] https.request(opts):');
debug(finalOpts);
req = https.request(finalOpts, onResponse);
requester = https;
} else if ('http:' === finalOpts.protocol) {
// https://nodejs.org/api/http.html#http_http_request_options_callback
debug("\n[urequest] http.request(opts):");
debug('\n[urequest] http.request(opts):');
debug(finalOpts);
req = http.request(finalOpts, onResponse);
requester = http;
} else {
throw new Error("unknown protocol: '" + opts.uri.protocol + "'");
cb(new Error("unknown protocol: '" + opts.uri.protocol + "'"));
return;
}
req.on('error', function (e) {
cb(e);
if (form) {
debug("\n[urequest] '" + finalOpts.method + "' (request) form");
debug(formHeaders);
// generally uploads don't use Chunked Encoding (some systems have issues with it)
// and I don't want to do the work to calculate the content-lengths. This seems to work.
req = form.submit(finalOpts, function (err, resp) {
if (err) {
cb(err);
return;
}
onResponse(resp);
resp.resume();
});
//req = requester.request(finalOpts, onResponse);
//req.on('error', cb);
//form.pipe(req);
return;
}
if (opts.debug) {
console.debug('');
console.debug('[@root/request] Request Options:');
console.debug(finalOpts);
if (_body) {
console.debug('[@root/request] Request Body:');
console.debug(
opts.body || opts.form || opts.formData || opts.json
);
}
}
req = requester.request(finalOpts, onResponse);
req.once('error', cb);
if (_body) {
debug("\n[urequest] '" + finalOpts.method + "' (request) body");
debug(_body);
if ('function' === typeof _body.pipe) {
// used for chunked encoding
//req.write(_body);
_body.pipe(req);
_body.once('error', function (err) {
// https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options
// if the Readable stream emits an error during processing,
// the Writable destination is not closed automatically
_body.destroy();
req.destroy(err);
});
} else {
// used for known content-length
req.end(_body);
}
} else {
req.end();
}
}
function parseUrl(str) {
var obj = url.parse(str);
var paths;
if ('unix' !== (obj.hostname || obj.host || '').toLowerCase()) {
return obj;
}
obj.href = null;
obj.hostname = obj.host = null;
paths = (obj.pathname || obj.path || '').split(':');
obj.socketPath = paths.shift();
obj.pathname = obj.path = paths.join(':');
return obj;
}
function urequest(opts, cb) {
debug("\n[urequest] received options:");
debug('\n[urequest] received options:');
debug(opts);
var reqOpts = {};
// request.js behavior:
@ -230,10 +531,10 @@ function setDefaults(defs) {
if ('string' === typeof opts.url || 'string' === typeof opts.uri) {
if ('string' === typeof opts.url) {
reqOpts.url = opts.url;
reqOpts.uri = url.parse(opts.url);
reqOpts.uri = parseUrl(opts.url);
} else if ('string' === typeof opts.uri) {
reqOpts.url = opts.uri;
reqOpts.uri = url.parse(opts.uri);
reqOpts.uri = parseUrl(opts.uri);
}
} else {
if ('object' === typeof opts.uri) {
@ -247,60 +548,138 @@ function setDefaults(defs) {
}
}
if (
opts.body ||
(opts.json && true !== opts.json) ||
opts.form ||
opts.formData
) {
// TODO this is probably a deviation from request's API
// need to check and probably eliminate it
reqOpts.method = (reqOpts.method || 'POST').toUpperCase();
} else {
reqOpts.method = (reqOpts.method || 'GET').toUpperCase();
reqOpts.headers = reqOpts.headers || {};
}
if (!reqOpts.headers) {
reqOpts.headers = {};
}
// crazy case for easier testing
if (!hasHeader(reqOpts, 'CoNTeNT-TyPe')) {
if ((true === reqOpts.json && reqOpts.body)
|| (true !== reqOpts.json && reqOpts.json)) {
if (
(true === reqOpts.json && reqOpts.body) ||
(true !== reqOpts.json && reqOpts.json)
) {
reqOpts.headers['Content-Type'] = 'application/json';
}
}
if (opts.debug) {
reqOpts.debug = opts.debug;
}
return urequestHelper(reqOpts, cb);
}
urequest.defaults = function (_defs) {
function smartPromisify(fn) {
return function (opts) {
var cb;
if ('function' === typeof arguments[1]) {
cb = Array.prototype.slice.call(arguments)[1];
return fn(opts, cb);
}
return new Promise(function (resolve, reject) {
fn(opts, function (err, resp) {
if (err) {
err._response = resp;
reject(err);
return;
}
resolve(resp);
});
});
};
}
var smartUrequest = smartPromisify(urequest);
smartUrequest.defaults = function (_defs) {
_defs = mergeOrDelete(defs, _defs);
return setDefaults(_defs);
};
[ 'get', 'put', 'post', 'patch', 'delete', 'head', 'options' ].forEach(function (method) {
urequest[method] = function (obj) {
['get', 'put', 'post', 'patch', 'delete', 'head', 'options'].forEach(
function (method) {
smartUrequest[method] = smartPromisify(function (obj, cb) {
if ('string' === typeof obj) {
obj = { url: obj };
}
obj.method = method.toUpperCase();
urequest(obj);
};
urequest(obj, cb);
});
urequest.del = urequest.delete;
}
);
smartUrequest.del = urequest.delete;
return urequest;
return smartUrequest;
}
var nodeUa =
'@root+request/' +
pkg.version +
' ' +
process.release.name +
'/' +
process.version +
' ' +
os.platform() +
'/' +
os.release() +
' ' +
os.type() +
'/' +
process.arch;
function getUserAgent(additional) {
// See https://tools.ietf.org/html/rfc8555#section-6.1
// And https://tools.ietf.org/html/rfc7231#section-5.5.3
// And https://community.letsencrypt.org/t/user-agent-flag-explained/3843/2
var ua = nodeUa;
if (additional) {
ua = additional + ' ' + ua;
}
return ua;
}
var _defaults = {
sendImmediately: true
, method: 'GET'
, headers: {}
, useQuerystring: false
, followRedirect: true
, followAllRedirects: false
, followOriginalHttpMethod: false
, maxRedirects: 10
, removeRefererHeader: false
//, encoding: undefined
, gzip: false
sendImmediately: true,
method: '',
headers: {},
useQuerystring: false,
followRedirect: true,
followAllRedirects: false,
followOriginalHttpMethod: false,
maxRedirects: 10,
removeRefererHeader: false,
// encoding: undefined,
// stream: false, // TODO allow a stream?
gzip: false
//, body: undefined
//, json: undefined
};
module.exports = setDefaults(_defaults);
module.exports._keys = Object.keys(_defaults).concat([
'encoding'
, 'body'
, 'json'
'encoding',
'stream',
'body',
'json',
'form',
'auth',
'formData',
'FormData',
'userAgent' // non-standard for request.js
]);
module.exports.debug = (-1 !== (process.env.NODE_DEBUG||'').split(/\s+/g).indexOf('urequest'));
module.exports.debug =
-1 !== (process.env.NODE_DEBUG || '').split(/\s+/g).indexOf('urequest');
debug("DEBUG ON for urequest");
debug('DEBUG ON for urequest');

5
package-lock.json generated Normal file
View File

@ -0,0 +1,5 @@
{
"name": "@root/request",
"version": "1.8.2",
"lockfileVersion": 1
}

View File

@ -1,8 +1,11 @@
{
"name": "@coolaj86/urequest",
"version": "1.1.2",
"description": "A lightweight drop-in replacement for request",
"name": "@root/request",
"version": "1.8.2",
"description": "A lightweight, zero-dependency drop-in replacement for request",
"main": "index.js",
"files": [
"lib"
],
"directories": {
"example": "examples"
},
@ -11,7 +14,7 @@
},
"repository": {
"type": "git",
"url": "https://git.ppl.family/ppl/urequest.js.git"
"url": "https://git.rootprojects.org/root/request.js.git"
},
"keywords": [
"request",

11
tests/tcp-listener.js Normal file
View File

@ -0,0 +1,11 @@
'use strict';
var net = require('net');
var server = net.createServer(function (socket) {
socket.on('data', function (chunk) {
console.info(chunk.toString('utf8'));
});
});
server.listen(3007, function () {
console.info('Listening on', this.address());
});