From 6b2b9607ecfccb2cd5f0b2bce739d33b535e7719 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 30 May 2018 22:48:32 -0600 Subject: [PATCH] update docs and cleanup files --- README.md | 133 ++++++++++++++++++++++---- index.js | 31 +++--- package.json | 2 +- input.json => test/input.json | 5 +- output.bin => test/output.bin | Bin 227 -> 255 bytes output.hexdump => test/output.hexdump | 0 test-pack.js => test/pack.js | 15 ++- test.js => test/parse.js | 26 ++--- sni.hello.bin => test/sni.hello.bin | Bin 9 files changed, 162 insertions(+), 50 deletions(-) rename input.json => test/input.json (64%) rename output.bin => test/output.bin (52%) rename output.hexdump => test/output.hexdump (100%) rename test-pack.js => test/pack.js (57%) rename test.js => test/parse.js (79%) rename sni.hello.bin => test/sni.hello.bin (100%) diff --git a/README.md b/README.md index 30e7d26..eaf6108 100644 --- a/README.md +++ b/README.md @@ -2,32 +2,123 @@ | Sponsored by [ppl](https://ppl.family) | -A strategy for packing and unpacking tunneled network messages (or any stream) in node.js +"The M-PROXY Protocol" for node.js -Examples +A strategy for packing and unpacking multiplexed streams. +Where you have distinct clients on one side trying to reach distinct servers on the other. + +``` +Browser <--\ /--> Device +Browser <---- M-PROXY Service ----> Device +Browser <--/ \--> Device +``` + +It's the kind of thing you'd use to build a poor man's VPN, or port-forward router. + +The M-PROXY Protocol +=================== + +This is similar to "The PROXY Protocol" (a la HAProxy), but desgined for multiplexed tls, http, tcp, and udp +tunneled over arbitrary streams (such as WebSockets). + +It also has a backchannel for communicating with the proxy itself. + +Each message has a header with a socket identifier (family, addr, port), and may have additional information. + +``` +,
,,,,, +``` + +``` +<254><45>IPv4,127.0.1.1,4321,199,https,443,example.com +``` + +``` +version (8 bits) 254 is version 1 + +header length (8 bits) the remaining length of the header before data begins + + These values are used to identify a specific client among many +socket family (string) the IPv4 or IPv6 connection from a client +socket address (string) the x.x.x.x remote address of the client +socket port (string) the 1-65536 source port of the remote client + +data length (string) the number of bytes in the wrapped packet, in case the network re-chunks the packet + + These optional values can be very useful at the start of a new connection +service name (string) Either a standard service name (port + protocol), such as 'https' + as listed in /etc/services, otherwise 'tls', 'tcp', or 'udp' for generics + Also 'control' is used for messages to the proxy (such as 'pause' events) +service port (string) The listening port, such as 443. Useful for non-standard or dynamic services. +host or server name (string) Useful for services that can be routed by name, such as http, https, smtp, and dns. +``` + +v1 is text-based. Future versions may be binary. + +API +=== ```js var Packer = require('proxy-packer'); +``` -Packer.create({ - onmessage: function (msg) { - // msg = { family, address, port, service, data }; - } -, onend: function (msg) { - // msg = { family, address, port }; - } -, onerror: function (err) { - // err = { message, family, address, port }; - } -}); +```js +unpacker = Packer.create(handlers); // Create a state machine for unpacking -var chunk = Packer.pack(address, data, service); -var addr = Packer.socketToAddr(socket); -var id = Packer.addrToId(address); -var id = Packer.socketToId(socket); +handlers.oncontrol = function (tun) { } // for communicating with the proxy + // tun.data is an array + // '[ -1, "[Error] bad hello" ]' + // '[ 0, "[Error] out-of-band error message" ]' + // '[ 1, "hello", 254, [ "add_token", "delete_token" ] ]' + // '[ 1, "add_token" ]' + // '[ 1, "delete_token" ]' -var myDuplex = Packer.Stream.create(socketOrStream); +handlers.onmessage = function (tun) { } // a client has sent a message + // tun = { family, address, port, data + // , service, serviceport, name }; +handlers.onpause = function (tun) { } // proxy requests to pause upload to a client + // tun = { family, address, port }; + +handlers.onresume = function (tun) { } // proxy requests to resume upload to a client + // tun = { family, address, port }; + +handlers.onend = function (tun) { } // proxy requests to close a client's socket + // tun = { family, address, port }; + +handlers.onerror = function (err) { } // proxy is relaying a client's error + // err = { message, family, address, port }; +``` + + + +```js +var chunk = Packer.pack(tun, data); // Add M-PROXY header to data + // tun = { family, address, port + // , service, serviceport, name } + +var addr = Packer.socketToAddr(socket); // Probe raw, raw socket for address info + +var id = Packer.addrToId(address); // Turn M-PROXY address info into a deterministic id + +var id = Packer.socketToId(socket); // Turn raw, raw socket info into a deterministic id +``` + +## API Helpers + +```js +var socket = Packer.Stream.wrapSocket(socketOrStream); // workaround for https://github.com/nodejs/node/issues/8854 + // which was just closed recently, but probably still needs + // something more like this (below) to work as intended + // https://github.com/findhit/proxywrap/blob/master/lib/proxywrap.js +``` + +```js var myTransform = Packer.Transform.create({ address: { family: '...' @@ -45,7 +136,7 @@ If you want to write a compatible packer, just make sure that for any given inpu you get the same output as the packer does. ```bash -node test-pack.js input.json output.bin +node test/pack.js input.json output.bin hexdump output.bin ``` @@ -57,8 +148,10 @@ Where `input.json` looks something like this: , "address": { "family": "IPv4" , "address": "127.0.1.1" - , "port": 443 + , "port": 4321 , "service": "foo" + , "serviceport": 443 + , "name": 'example.com' } , "filepath": "./sni.tcp.bin" } diff --git a/index.js b/index.js index 709ead5..b9ecd22 100644 --- a/index.js +++ b/index.js @@ -120,6 +120,8 @@ Packer.create = function (opts) { machine.port = machine._headers[2]; machine.bodyLen = parseInt(machine._headers[3], 10) || 0; machine.service = machine._headers[4]; + machine.serviceport = machine._headers[5]; + machine.name = machine._headers[6]; //console.log('machine.service', machine.service); return true; @@ -148,11 +150,13 @@ Packer.create = function (opts) { } } - msg.family = machine.family; - msg.address = machine.address; - msg.port = machine.port; - msg.service = machine.service; - msg.data = data; + msg.family = machine.family; + msg.address = machine.address; + msg.port = machine.port; + msg.service = machine.service; + msg.serviceport = machine.serviceport; + msg.name = machine.name; + msg.data = data; if (machine.emit) { machine.emit(serviceEvents[msg.service] || serviceEvents.default); @@ -182,7 +186,7 @@ Packer.create = function (opts) { return machine; }; -Packer.pack = function (address, data, service) { +Packer.pack = function (meta, data, service) { data = data || Buffer.from(' '); if (!Buffer.isBuffer(data)) { data = new Buffer(JSON.stringify(data)); @@ -192,7 +196,7 @@ Packer.pack = function (address, data, service) { } if (service && service !== 'control') { - address.service = service; + meta.service = service; } var version = 1; @@ -202,13 +206,14 @@ Packer.pack = function (address, data, service) { } else { header = Buffer.from([ - address.family, address.address, address.port, data.byteLength, (address.service || '') + meta.family, meta.address, meta.port, data.byteLength, + (meta.service || ''), (meta.serviceport || ''), (meta.name || '') ].join(',')); } - var meta = Buffer.from([ 255 - version, header.length ]); - var buf = Buffer.alloc(meta.byteLength + header.byteLength + data.byteLength); + var metaBuf = Buffer.from([ 255 - version, header.length ]); + var buf = Buffer.alloc(metaBuf.byteLength + header.byteLength + data.byteLength); - meta.copy(buf, 0); + metaBuf.copy(buf, 0); header.copy(buf, 2); data.copy(buf, 2 + header.byteLength); @@ -323,6 +328,8 @@ function MyTransform(options) { } this.__my_addr = options.address; this.__my_service = options.service; + this.__my_serviceport = options.serviceport; + this.__my_name = options.name; Transform.call(this, options); } util.inherits(MyTransform, Transform); @@ -331,6 +338,8 @@ MyTransform.prototype._transform = function (data, encoding, callback) { var address = this.__my_addr; address.service = address.service || this.__my_service; + address.serviceport = address.serviceport || this.__my_serviceport; + address.name = address.name || this.__my_name; this.push(Packer.pack(address, data)); callback(); }; diff --git a/package.json b/package.json index 65b8f12..8af11ff 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "proxy-packer", - "version": "1.4.2", + "version": "1.4.3", "description": "A strategy for packing and unpacking a proxy stream (i.e. packets through a tunnel). Handles multiplexed and tls connections. Used by telebit and telebitd.", "main": "index.js", "scripts": { diff --git a/input.json b/test/input.json similarity index 64% rename from input.json rename to test/input.json index efc13d8..8c0751c 100644 --- a/input.json +++ b/test/input.json @@ -2,8 +2,9 @@ , "address": { "family": "IPv4" , "address": "127.0.1.1" - , "port": 443 - , "service": "foo" + , "port": 4321 + , "service": "https" + , "serviceport": 443 } , "filepath": "./sni.hello.bin" } diff --git a/output.bin b/test/output.bin similarity index 52% rename from output.bin rename to test/output.bin index 4c38fa6c2489eb004f763191b40c9e40c2f94f7a..1979205a7c5f8ff8f448fdc47a851189bdb7e4c9 100644 GIT binary patch delta 64 zcmaFN_@B|@pP6SsnTd{}k-46Mo}r$hj)}36p^l-YrA|gkNkOrWiHWgJL4I~>ZeoF6 TMruw@Vp3*_UUGi!M496Nfie`% delta 36 rcmey*_?S`VpOj}nnTd{}k-46Mo}r$hj){q}j-jQcPFjBcM2q79&4&uq diff --git a/output.hexdump b/test/output.hexdump similarity index 100% rename from output.hexdump rename to test/output.hexdump diff --git a/test-pack.js b/test/pack.js similarity index 57% rename from test-pack.js rename to test/pack.js index b0e15b4..ba1f822 100644 --- a/test-pack.js +++ b/test/pack.js @@ -4,24 +4,30 @@ var fs = require('fs'); var infile = process.argv[2]; var outfile = process.argv[3]; +var sni = require('sni'); if (!infile || !outfile) { console.error("Usage:"); - console.error("node test-pack.js input.json output.bin"); + console.error("node test/pack.js test/input.json test/output.bin"); process.exit(1); return; } +var path = require('path'); var json = JSON.parse(fs.readFileSync(infile, 'utf8')); -var data = require('fs').readFileSync(json.filepath, null); -var Packer = require('./index.js'); +var data = require('fs').readFileSync(path.resolve(path.dirname(infile), json.filepath), null); +var Packer = require('../index.js'); + +var servername = sni(data); +var m = data.toString().match(/(?:^|[\r\n])Host: ([^\r\n]+)[\r\n]*/im); +var hostname = (m && m[1].toLowerCase() || '').split(':')[0]; /* function pack() { var version = json.version; var address = json.address; var header = address.family + ',' + address.address + ',' + address.port + ',' + data.byteLength - + ',' + (address.service || '') + + ',' + (address.service || '') + ',' + (address.serviceport || '') + ',' + (servername || hostname || '') ; var buf = Buffer.concat([ Buffer.from([ 255 - version, header.length ]) @@ -31,6 +37,7 @@ function pack() { } */ +json.address.name = servername || hostname; var buf = Packer.pack(json.address, data); fs.writeFileSync(outfile, buf, null); console.log("wrote " + buf.byteLength + " bytes to '" + outfile + "' ('hexdump " + outfile + "' to inspect)"); diff --git a/test.js b/test/parse.js similarity index 79% rename from test.js rename to test/parse.js index cb4b9a8..25ab0b0 100644 --- a/test.js +++ b/test/parse.js @@ -1,16 +1,18 @@ 'use strict'; var sni = require('sni'); -var hello = require('fs').readFileSync('./sni.hello.bin'); +var hello = require('fs').readFileSync(__dirname + '/sni.hello.bin'); var version = 1; var address = { family: 'IPv4' , address: '127.0.1.1' -, port: 443 -, service: 'foo' +, port: 4321 +, service: 'foo-https' +, serviceport: 443 +, name: 'foo-pokemap.hellabit.com' }; var header = address.family + ',' + address.address + ',' + address.port + ',' + hello.byteLength - + ',' + (address.service || '') + + ',' + (address.service || '') + ',' + (address.serviceport || '') + ',' + (address.name || '') ; var buf = Buffer.concat([ Buffer.from([ 255 - version, header.length ]) @@ -20,21 +22,21 @@ var buf = Buffer.concat([ var services = { 'ssh': 22, 'http': 4080, 'https': 8443 }; var clients = {}; var count = 0; -var packer = require('./'); +var packer = require('../'); var machine = packer.create({ - onmessage: function (opts) { - var id = opts.family + ',' + opts.address + ',' + opts.port; + onmessage: function (tun) { + var id = tun.family + ',' + tun.address + ',' + tun.port; var service = 'https'; var port = services[service]; - var servername = sni(opts.data); + var servername = sni(tun.data); console.log(''); console.log('[onMessage]'); - if (!opts.data.equals(hello)) { + if (!tun.data.equals(hello)) { throw new Error("'data' packet is not equal to original 'hello' packet"); } - console.log('all', opts.data.byteLength, 'bytes are equal'); - console.log('src:', opts.family, opts.address + ':' + opts.port); + console.log('all', tun.data.byteLength, 'bytes are equal'); + console.log('src:', tun.family, tun.address + ':' + tun.port + ':' + tun.serviceport); console.log('dst:', 'IPv4 127.0.0.1:' + port); if (!clients[id]) { @@ -42,7 +44,7 @@ var machine = packer.create({ if (!servername) { throw new Error("no servername found for '" + id + "'"); } - console.log("servername: '" + servername + "'"); + console.log("servername: '" + servername + "'", tun.name); } count += 1; diff --git a/sni.hello.bin b/test/sni.hello.bin similarity index 100% rename from sni.hello.bin rename to test/sni.hello.bin