request.js/index.js

'use strict';

var http = require('http');
var https = require('https');
var url = require('url');
var os = require('os');
var pkg = require('./package.json');
var fs = require('fs'); // only for streams

function debug() {
    if (module.exports.debug) {
        console.log.apply(console, arguments);
    }
}

function mergeOrDelete(defaults, updates) {
    Object.keys(defaults).forEach(function (key) {
        if (!(key in updates)) {
            updates[key] = defaults[key];
            return;
        }

        // neither accept the prior default nor define an explicit value
        // CRDT probs...
        if ('undefined' === typeof updates[key]) {
            delete updates[key];
        } else if (
            'object' === typeof defaults[key] &&
            'object' === typeof updates[key]
        ) {
            updates[key] = mergeOrDelete(defaults[key], updates[key]);
        }
    });

    return updates;
}

// retrieves an existing header, case-sensitive
function getHeaderName(reqOpts, header) {
    var headerNames = {};
    Object.keys(reqOpts.headers).forEach(function (casedName) {
        headerNames[casedName.toLowerCase()] = casedName;
    });
    // returns the key, which in erroneous cases could be an empty string
    return headerNames[header.toLowerCase()];
}
// returns whether or not a header exists, case-insensitive
function hasHeader(reqOpts, header) {
    return 'undefined' !== typeof getHeaderName(reqOpts, header);
}

function toJSONifier(keys) {
    return function () {
        var obj = {};
        var me = this;

        keys.forEach(function (key) {
            if (me[key] && 'function' === typeof me[key].toJSON) {
                obj[key] = me[key].toJSON();
            } else {
                obj[key] = me[key];
            }
        });

        return obj;
    };
}

function setupPipe(resp, opts) {
    // make the response await-able
    var resolve;
    var reject;
    var p = new Promise(function (_resolve, _reject) {
        resolve = _resolve;
        reject = _reject;
    });

    // or an existing write stream
    if ('function' === typeof opts.stream.pipe) {
        if (opts.debug) {
            console.debug('[@root/request] stream piped');
        }
        resp.pipe(opts.stream);
    }
    resp.once('error', function (e) {
        if (opts.debug) {
            console.debug("[@root/request] stream 'error'");
            console.error(e.stack);
        }
        resp.destroy();
        if ('function' === opts.stream.destroy) {
            opts.stream.destroy(e);
        }
        reject(e);
    });
    resp.once('end', function () {
        if (opts.debug) {
            console.debug("[@root/request] stream 'end'");
        }
        if ('function' === opts.stream.destroy) {
            opts.stream.end();
            // this will close the stream (i.e. sync to disk)
            opts.stream.destroy();
        }
    });
    resp.once('close', function () {
        if (opts.debug) {
            console.debug("[@root/request] stream 'close'");
        }
        resolve();
    });
    return p;
}

function handleResponse(resp, opts, cb) {
    // body can be buffer, string, or json
    if (null === opts.encoding) {
        resp._body = [];
    } else {
        resp.body = '';
    }
    resp._bodyLength = 0;
    resp.on('readable', function () {
        var chunk;
        while ((chunk = resp.read())) {
            if ('string' === typeof resp.body) {
                resp.body += chunk.toString(opts.encoding);
            } else {
                resp._body.push(chunk);
                resp._bodyLength += chunk.length;
            }
        }
    });
    resp.once('end', function () {
        if ('string' !== typeof resp.body) {
            if (1 === resp._body.length) {
                resp.body = resp._body[0];
            } else {
                resp.body = Buffer.concat(resp._body, resp._bodyLength);
            }
            resp._body = null;
        }
        if (opts.json && 'string' === typeof resp.body) {
            // TODO I would parse based on Content-Type
            // but request.js doesn't do that.
            try {
                resp.body = JSON.parse(resp.body);
            } catch (e) {
                // ignore
            }
        }

        debug('\n[urequest] resp.toJSON():');
        if (module.exports.debug) {
            debug(resp.toJSON());
        }
        if (opts.debug) {
            console.debug('[@root/request] Response Body:');
            console.debug(resp.body);
        }
        cb(null, resp, resp.body);
    });
}

function setDefaults(defs) {
    defs = defs || {};

    function urequestHelper(opts, cb) {
        debug('\n[urequest] processed options:');
        debug(opts);

        var req;
        var finalOpts = {};

        // allow specifying a file
        if ('string' === typeof opts.stream) {
            if (opts.debug) {
                console.debug('[@root/request] creating file write stream');
            }
            try {
                opts.stream = fs.createWriteStream(opts.stream);
            } catch (e) {
                cb(e);
                return;
            }
        }

        function onResponse(resp) {
            var followRedirect;

            Object.keys(defs).forEach(function (key) {
                if (key in opts && 'undefined' !== typeof opts[key]) {
                    return;
                }
                opts[key] = defs[key];
            });
            followRedirect = opts.followRedirect;

            // copied from WHATWG fetch
            resp.ok = false;
            if (resp.statusCode >= 200 && resp.statusCode < 300) {
                resp.ok = true;
            }

            resp.toJSON = toJSONifier([
                'statusCode',
                'body',
                'ok',
                'headers',
                'request'
            ]);

            resp.request = req;
            resp.request.uri = url.parse(opts.url);
            //resp.request.method = opts.method;
            resp.request.headers = finalOpts.headers;
            resp.request.toJSON = toJSONifier(['uri', 'method', 'headers']);
            if (opts.debug) {
                console.debug('[@root/request] Response Headers:');
                console.debug(resp.toJSON());
            }

            if (
                followRedirect &&
                resp.headers.location &&
                -1 !== [301, 302, 307, 308].indexOf(resp.statusCode)
            ) {
                debug('Following redirect: ' + resp.headers.location);
                if ('GET' !== opts.method && !opts.followAllRedirects) {
                    followRedirect = false;
                }
                if (opts._redirectCount >= opts.maxRedirects) {
                    followRedirect = false;
                }
                if ('function' === opts.followRedirect) {
                    if (!opts.followRedirect(resp)) {
                        followRedirect = false;
                    }
                }
                if (followRedirect) {
                    if (!opts.followOriginalHttpMethod) {
                        opts.method = 'GET';
                        opts.body = null;
                        delete opts.headers[
                            getHeaderName(opts, 'Content-Length')
                        ];
                        delete opts.headers[
                            getHeaderName(opts, 'Transfer-Encoding')
                        ];
                    }
                    if (opts.removeRefererHeader && opts.headers) {
                        delete opts.headers.referer;
                    }
                    // TODO needs baseUrl, maybe test for host / socketPath?
                    opts.url = resp.headers.location;
                    opts.uri = url.parse(opts.url);
                    return urequestHelper(opts, cb);
                }
            }

            if (opts.stream) {
                resp.stream = setupPipe(resp, opts);
                // can be string, buffer, or json... why not an async function too?
                resp.stream.body = async function () {
                    handleResponse(resp, opts, cb);
                    await resp.stream;
                    return resp.body;
                };
                cb(null, resp);
                return;
            }

            handleResponse(resp, opts, cb);
        }

        var _body;
        var MyFormData;
        var form;
        var formHeaders;
        var requester;

        if (opts.body) {
            if (true === opts.json && 'string' !== typeof opts.body) {
                _body = JSON.stringify(opts.body);
            } else {
                _body = opts.body;
            }
        } else if (opts.json && true !== opts.json) {
            _body = JSON.stringify(opts.json);
        } else if (opts.form) {
            _body = Object.keys(opts.form)
                .filter(function (key) {
                    if ('undefined' !== typeof opts.form[key]) {
                        return true;
                    }
                })
                .map(function (key) {
                    return (
                        encodeURIComponent(key) +
                        '=' +
                        encodeURIComponent(String(opts.form[key]))
                    );
                })
                .join('&');
            opts.headers['Content-Type'] = 'application/x-www-form-urlencoded';
        }
        if ('string' === typeof _body) {
            _body = Buffer.from(_body);
        }

        Object.keys(opts.uri).forEach(function (key) {
            finalOpts[key] = opts.uri[key];
        });

        // A bug should be raised if request does it differently,
        // but I think we're supposed to pass all acceptable options
        // on to the raw http request
        [
            'family',
            'host',
            'localAddress',
            'agent',
            'createConnection',
            'timeout',
            'setHost'
        ].forEach(function (key) {
            if (key in opts) {
                finalOpts[key] = opts[key];
            }
        });

        finalOpts.method = opts.method;
        finalOpts.headers = JSON.parse(JSON.stringify(opts.headers));

        var uaHeader = getHeaderName(finalOpts, 'User-Agent') || 'User-Agent';
        // set a default user-agent
        if (!finalOpts.headers[uaHeader]) {
            if (false === finalOpts.headers[uaHeader]) {
                delete finalOpts.headers[uaHeader];
            } else {
                finalOpts.headers[uaHeader] = getUserAgent(opts.userAgent);
            }
        }
        if (_body) {
            // Most APIs expect (or require) Content-Length except in the case of multipart uploads
            // Transfer-Encoding: Chunked (the default) is generally only well-supported downstream
            if (
                'undefined' !== typeof _body.byteLength ||
                'undefined' !== typeof _body.length
            ) {
                finalOpts.headers['Content-Length'] =
                    _body.byteLength || _body.length;
            }
        }
        if (opts.auth) {
            // if opts.uri specifies auth it will be parsed by url.parse and passed directly to the http module
            if ('string' !== typeof opts.auth) {
                opts.auth =
                    (opts.auth.user || opts.auth.username || '') +
                    ':' +
                    (opts.auth.pass || opts.auth.password || '');
            }
            if ('string' === typeof opts.auth) {
                finalOpts.auth = opts.auth;
            }
            if (false === opts.sendImmediately) {
                console.warn(
                    '[Warn] setting `sendImmediately: false` is not yet supported. Please open an issue if this is an important feature that you need.'
                );
            }
            // [request-compat]
            if (opts.auth.bearer) {
                // having a shortcut for base64 encoding makes sense, but this? Eh, whatevs...
                finalOpts.header.Authorization = 'Bearer ' + opts.auth.bearer;
            }
        }
        if (opts.formData) {
            try {
                MyFormData = opts.FormData || require('form-data');
                // potential options https://github.com/felixge/node-combined-stream/blob/master/lib/combined_stream.js#L7-L15
            } catch (e) {
                console.error(
                    'urequest does not include extra dependencies by default'
                );
                console.error(
                    "if you need to use 'form-data' you may install it, like so:"
                );
                console.error('  npm install --save form-data');
                cb(e);
                return;
            }
            try {
                form = new MyFormData();
                Object.keys(opts.formData).forEach(function (key) {
                    function add(key, data, opts) {
                        if (data.value) {
                            opts = data.options;
                            data = data.value;
                        }
                        form.append(key, data, opts);
                    }
                    if (Array.isArray(opts.formData[key])) {
                        opts.formData[key].forEach(function (data) {
                            add(key, data);
                        });
                    } else {
                        add(key, opts.formData[key]);
                    }
                });
            } catch (e) {
                cb(e);
                return;
            }
            formHeaders = form.getHeaders();
            Object.keys(formHeaders).forEach(function (header) {
                finalOpts.headers[header] = formHeaders[header];
            });
        }

        // TODO support unix sockets
        if ('https:' === finalOpts.protocol) {
            // https://nodejs.org/api/https.html#https_https_request_options_callback
            debug('\n[urequest] https.request(opts):');
            debug(finalOpts);
            requester = https;
        } else if ('http:' === finalOpts.protocol) {
            // https://nodejs.org/api/http.html#http_http_request_options_callback
            debug('\n[urequest] http.request(opts):');
            debug(finalOpts);
            requester = http;
        } else {
            cb(new Error("unknown protocol: '" + opts.uri.protocol + "'"));
            return;
        }

        if (form) {
            debug("\n[urequest] '" + finalOpts.method + "' (request) form");
            debug(formHeaders);
            // generally uploads don't use Chunked Encoding (some systems have issues with it)
            // and I don't want to do the work to calculate the content-lengths. This seems to work.
            req = form.submit(finalOpts, function (err, resp) {
                if (err) {
                    cb(err);
                    return;
                }
                onResponse(resp);
                resp.resume();
            });
            //req = requester.request(finalOpts, onResponse);
            //req.on('error', cb);
            //form.pipe(req);
            return;
        }

        if (opts.debug) {
            console.debug('');
            console.debug('[@root/request] Request Options:');
            console.debug(finalOpts);
            if (_body) {
                console.debug('[@root/request] Request Body:');
                console.debug(
                    opts.body || opts.form || opts.formData || opts.json
                );
            }
        }
        req = requester.request(finalOpts, onResponse);
        req.once('error', cb);

        if (_body) {
            debug("\n[urequest] '" + finalOpts.method + "' (request) body");
            debug(_body);
            if ('function' === typeof _body.pipe) {
                // used for chunked encoding
                _body.pipe(req);
                _body.once('error', function (err) {
                    // https://nodejs.org/api/stream.html#stream_readable_pipe_destination_options
                    // if the Readable stream emits an error during processing,
                    // the Writable destination is not closed automatically
                    _body.destroy();
                    req.destroy(err);
                });
            } else {
                // used for known content-length
                req.end(_body);
            }
        } else {
            req.end();
        }
    }

    function parseUrl(str) {
        var obj = url.parse(str);
        var paths;
        if ('unix' !== (obj.hostname || obj.host || '').toLowerCase()) {
            return obj;
        }

        obj.href = null;
        obj.hostname = obj.host = null;

        paths = (obj.pathname || obj.path || '').split(':');

        obj.socketPath = paths.shift();
        obj.pathname = obj.path = paths.join(':');

        return obj;
    }

    function urequest(opts, cb) {
        debug('\n[urequest] received options:');
        debug(opts);
        var reqOpts = {};
        // request.js behavior:
        // encoding: null + json ? unknown
        // json => attempt to parse, fail silently
        // encoding => buffer.toString(encoding)
        // null === encoding => Buffer.concat(buffers)
        if ('string' === typeof opts) {
            opts = { url: opts };
        }

        module.exports._keys.forEach(function (key) {
            if (key in opts && 'undefined' !== typeof opts[key]) {
                reqOpts[key] = opts[key];
            } else if (key in defs) {
                reqOpts[key] = defs[key];
            }
        });

        // TODO url.resolve(defs.baseUrl, opts.url);
        if ('string' === typeof opts.url || 'string' === typeof opts.uri) {
            if ('string' === typeof opts.url) {
                reqOpts.url = opts.url;
                reqOpts.uri = parseUrl(opts.url);
            } else if ('string' === typeof opts.uri) {
                reqOpts.url = opts.uri;
                reqOpts.uri = parseUrl(opts.uri);
            }
        } else {
            if ('object' === typeof opts.uri) {
                reqOpts.url = url.format(opts.uri);
                reqOpts.uri = opts.uri;
                //reqOpts.uri = url.parse(reqOpts.uri);
            } else if ('object' === typeof opts.url) {
                reqOpts.url = url.format(opts.url);
                reqOpts.uri = opts.url;
                //reqOpts.uri = url.parse(reqOpts.url);
            }
        }

        if (
            opts.body ||
            (opts.json && true !== opts.json) ||
            opts.form ||
            opts.formData
        ) {
            // TODO this is probably a deviation from request's API
            // need to check and probably eliminate it
            reqOpts.method = (reqOpts.method || 'POST').toUpperCase();
        } else {
            reqOpts.method = (reqOpts.method || 'GET').toUpperCase();
        }
        if (!reqOpts.headers) {
            reqOpts.headers = {};
        }

        // crazy case for easier testing
        if (!hasHeader(reqOpts, 'CoNTeNT-TyPe')) {
            if (
                (true === reqOpts.json && reqOpts.body) ||
                (true !== reqOpts.json && reqOpts.json)
            ) {
                reqOpts.headers['Content-Type'] = 'application/json';
            }
        }

        if (opts.debug) {
            reqOpts.debug = opts.debug;
        }
        return urequestHelper(reqOpts, cb);
    }

    function smartPromisify(fn) {
        return function (opts) {
            var cb;
            if ('function' === typeof arguments[1]) {
                cb = Array.prototype.slice.call(arguments)[1];
                return fn(opts, cb);
            }
            return new Promise(function (resolve, reject) {
                fn(opts, function (err, resp) {
                    if (err) {
                        err._response = resp;
                        reject(err);
                        return;
                    }
                    resolve(resp);
                });
            });
        };
    }

    var smartUrequest = smartPromisify(urequest);

    smartUrequest.defaults = function (_defs) {
        _defs = mergeOrDelete(defs, _defs);
        return setDefaults(_defs);
    };
    ['get', 'put', 'post', 'patch', 'delete', 'head', 'options'].forEach(
        function (method) {
            smartUrequest[method] = smartPromisify(function (obj, cb) {
                if ('string' === typeof obj) {
                    obj = { url: obj };
                }
                obj.method = method.toUpperCase();
                urequest(obj, cb);
            });
        }
    );
    smartUrequest.del = urequest.delete;

    return smartUrequest;
}

var nodeUa =
    '@root+request/' +
    pkg.version +
    ' ' +
    process.release.name +
    '/' +
    process.version +
    ' ' +
    os.platform() +
    '/' +
    os.release() +
    ' ' +
    os.type() +
    '/' +
    process.arch;
function getUserAgent(additional) {
    // See https://tools.ietf.org/html/rfc8555#section-6.1
    // And https://tools.ietf.org/html/rfc7231#section-5.5.3
    // And https://community.letsencrypt.org/t/user-agent-flag-explained/3843/2

    var ua = nodeUa;
    if (additional) {
        ua = additional + ' ' + ua;
    }

    return ua;
}

var _defaults = {
    sendImmediately: true,
    method: '',
    headers: {},
    useQuerystring: false,
    followRedirect: true,
    followAllRedirects: false,
    followOriginalHttpMethod: false,
    maxRedirects: 10,
    removeRefererHeader: false,
    // encoding: undefined,
    // stream: false, // TODO allow a stream?
    gzip: false
    //, body: undefined
    //, json: undefined
};
module.exports = setDefaults(_defaults);

module.exports._keys = Object.keys(_defaults).concat([
    'encoding',
    'stream',
    'body',
    'json',
    'form',
    'auth',
    'formData',
    'FormData',
    'userAgent' // non-standard for request.js
]);
module.exports.debug =
    -1 !== (process.env.NODE_DEBUG || '').split(/\s+/g).indexOf('urequest');

debug('DEBUG ON for urequest');