move tls-unwrapping (it may be sharable with client)

There was a problem that prevented socket events like close and error from
getting through our duplex and to the websocket so it could close

.jshintrc

@@ -0,0 +1,16 @@
+{ "node": true
+, "browser": true
+, "jquery": true
+, "strict": true
+, "indent": 2
+, "onevar": true
+, "laxcomma": true
+, "laxbreak": true
+, "eqeqeq": true
+, "immed": true
+, "undef": true
+, "unused": true
+, "latedef": true
+, "curly": true
+, "trailing": true
+}

README.md

@@ -1,21 +1,7 @@
-<!-- BANNER_TPL_BEGIN -->
-
-About Daplie: We're taking back the Internet!
---------------
-
-Down with Google, Apple, and Facebook!
-
-We're re-decentralizing the web and making it read-write again - one home cloud system at a time.
-
-Tired of serving the Empire? Come join the Rebel Alliance:
-
-<a href="mailto:jobs@daplie.com">jobs@daplie.com</a> | [Invest in Daplie on Wefunder](https://daplie.com/invest/) | [Pre-order Cloud](https://daplie.com/preorder/), The World's First Home Server for Everyone
-
-<!-- BANNER_TPL_END -->
-
+| Sponsored by [ppl](https://ppl.family) | **tunnel-server.js** | [tunnel-client.js](https://git.coolaj86.com/coolaj86/tunnel-client.js) |
 # stunneld.js
 
-A server that works in combination with [stunnel.js](https://github.com/Daplie/node-tunnel-client)
+A server that works in combination with [stunnel.js](https://git.coolaj86.com/coolaj86/tunnel-client.js)
 to allow you to serve http and https from any computer, anywhere through a secure tunnel.
 
 CLI
@@ -27,9 +13,20 @@ Installs as `stunnel.js` with the alias `jstunnel`
 ### Install
 
 ```bash
-npm install -g stunnel
+npm install -g stunneld
 ```
 
+Then `dist/etc/systemd/system/stunneld.service` should be copied to `/etc/systemd/system/stunneld.service` and
+the ARGUMENTS, such as SECRET, MUST BE CHANGED.
+
+*TODO*: make `--config /path/to/config` the only argument (and have the secret auto-generated on first run?)
+
+## Note: Use node.js v8.x
+
+There is a bug in node v9.x that causes stunneld to crash.
+
+https://github.com/nodejs/node/issues/20241
+
 ### Advanced Usage
 
 How to use `stunnel.js` with your own instance of `stunneld.js`:
@@ -62,11 +59,11 @@ but those generally cost $5 - $20 / month and so it's probably
 cheaper to purchase data transfer (which we supply, obviously),
 which is only $1/month for most people.
 
-Just use the client ([stunnel.js](https://github.com/Daplie/node-tunnel-client))
-with Daplie's tunneling service (the default) and save yourself the monthly fee
+Just use the client ([stunnel.js](https://git.coolaj86.com/coolaj86/tunnel-client.js))
+with this tunneling service (the default) and save yourself the monthly fee
 by only paying for the data you need.
 
-	* Daplie Tunnel (zero setup)
+	* Node WS Tunnel (zero setup)
 	* Heroku (zero cost)
 	* Chunk Host (best deal per TB/month)
 

bin/generate-secret.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+'use strict'
+
+console.log(require('crypto').randomBytes(16).toString('hex'));

bin/install-stunneld-js.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+rm -rf ./node-installer.sh
+curl -fsSL bit.ly/node-installer -o ./node-installer.sh
+bash ./node-installer.sh --dev-deps
+
+git clone https://git.coolaj86.com/coolaj86/tunnel-server.js.git
+pushd tunnel-server.js/
+  npm install
+  my_secret=$(node bin/generate-secret.js)
+  echo "Your secret is:\n\n\t"$my_secret
+  echo "node bin/server.js --servernames tunnel.example.com --secret $my_secret"
+popd

bin/stunneld.js

@@ -6,6 +6,15 @@ var pkg = require('../package.json');
 
 var program = require('commander');
 var stunneld = require('../wstunneld.js');
+var greenlock = require('greenlock');
+
+function collectServernames(val, memo) {
+  var lowerCase = val.split(/,/).map(function (servername) {
+    return servername.toLowerCase();
+  });
+
+  return memo.concat(lowerCase);
+}
 
 function collectProxies(val, memo) {
   var vals = val.split(/,/g);
@@ -56,22 +65,20 @@ function collectProxies(val, memo) {
 }
 
 function collectPorts(val, memo) {
-  memo = memo.concat(val.split(/,/g).filter(Boolean));
-  return memo;
+  return memo.concat(val.split(/,/g).map(Number).filter(Boolean));
 }
 
 program
   .version(pkg.version)
+  .option('--agree-tos', "Accept the Daplie and Let's Encrypt Terms of Service")
+  .option('--email <EMAIL>', "Email to use for Daplie and Let's Encrypt accounts")
   .option('--serve <URL>', 'comma separated list of <proto>:<//><servername>:<port> to which matching incoming http and https should forward (reverse proxy). Ex: https://john.example.com,tls:*:1337', collectProxies, [ ])
   .option('--ports <PORT>', 'comma separated list of ports on which to listen. Ex: 80,443,1337', collectPorts, [ ])
+  .option('--servernames <STRING>', 'comma separated list of servernames to use for the admin interface. Ex: tunnel.example.com,tunnel.example.net', collectServernames, [ ])
   .option('--secret <STRING>', 'the same secret used by stunneld (used for JWT authentication)')
   .parse(process.argv)
   ;
 
-if (!program.serve.length) {
-  throw new Error("must specify at least one server");
-}
-
 var portsMap = {};
 var servernamesMap = {};
 program.serve.forEach(function (proxy) {
@@ -80,27 +87,108 @@ program.serve.forEach(function (proxy) {
     portsMap[proxy.port] = true;
   }
 });
+program.servernames.forEach(function (name) {
+  servernamesMap[name] = true;
+});
 program.ports.forEach(function (port) {
   portsMap[port] = true;
 });
 
-var opts = {};
-opts.servernames = Object.keys(servernamesMap);
-opts.ports = Object.keys(portsMap);
-if (!opts.ports.length) {
-  opts.ports = [ 80, 443 ];
+program.servernames = Object.keys(servernamesMap);
+if (!program.servernames.length) {
+  throw new Error('You must give this server at least one servername for its admin interface. Example:\n\n\t--servernames tunnel.example.com,tunnel.example.net');
 }
 
-if (program.secret) {
-  opts.secret = program.secret;
-} else {
+program.ports = Object.keys(portsMap);
+if (!program.ports.length) {
+  program.ports = [ 80, 443 ];
+}
+
+if (!program.secret) {
   // TODO randomly generate and store in file?
-  console.warn("[SECURITY] using default --secret 'shhhhh'");
-  opts.secret = 'shhhhh';
+  console.warn("[SECURITY] you must provide --secret '" + require('crypto').randomBytes(16).toString('hex') + "'");
+  process.exit(1);
+  return;
 }
 
 // TODO letsencrypt
-opts.tlsOptions = require('localhost.daplie.com-certificates').merge({});
+program.tlsOptions = require('localhost.daplie.me-certificates').merge({});
+
+function approveDomains(opts, certs, cb) {
+  // This is where you check your database and associated
+  // email addresses with domains and agreements and such
+
+  // The domains being approved for the first time are listed in opts.domains
+  // Certs being renewed are listed in certs.altnames
+  if (certs) {
+    opts.domains = certs.altnames;
+  }
+  else {
+    if (-1 !== program.servernames.indexOf(opts.domain)) {
+      opts.email = program.email;
+      opts.agreeTos = program.agreeTos;
+    }
+  }
+
+  // NOTE: you can also change other options such as `challengeType` and `challenge`
+  // opts.challengeType = 'http-01';
+  // opts.challenge = require('le-challenge-fs').create({});
+
+  cb(null, { options: opts, certs: certs });
+}
+
+if (!program.email || !program.agreeTos) {
+  console.error("You didn't specify --email <EMAIL> and --agree-tos");
+  console.error("(required for ACME / Let's Encrypt / Greenlock TLS/SSL certs)");
+  console.error("");
+}
+else {
+  program.greenlock = greenlock.create({
+
+    version: 'draft-11'
+  , server: 'https://acme-v02.api.letsencrypt.org/directory'
+
+  , challenges: {
+      // TODO dns-01
+      'http-01': require('le-challenge-fs').create({ webrootPath: '/tmp/acme-challenges' })
+    }
+
+  , store: require('le-store-certbot').create({ webrootPath: '/tmp/acme-challenges' })
+
+  , email: program.email
+
+  , agreeTos: program.agreeTos
+
+  , approveDomains: approveDomains
+
+  //, approvedDomains: program.servernames
+
+  });
+}
+//program.tlsOptions.SNICallback = program.greenlock.httpsOptions.SNICallback;
+/*
+program.middleware = program.greenlock.middleware(function (req, res) {
+  res.end('Hello, World!');
+});
+*/
+
+require('../handlers').create(program); // adds directly to program for now...
+
+//require('cluster-store').create().then(function (store) {
+  //program.store = store;
+
+  var net = require('net');
+  var netConnHandlers = stunneld.create(program); // { tcp, ws }
+  var WebSocketServer = require('ws').Server;
+  var wss = new WebSocketServer({ server: (program.httpTunnelServer || program.httpServer) });
+	wss.on('connection', netConnHandlers.ws);
+  program.ports.forEach(function (port) {
+    var tcp3000 = net.createServer();
+    tcp3000.listen(port, function () {
+      console.log('listening on ' + port);
+    });
+    tcp3000.on('connection', netConnHandlers.tcp);
+  });
+//});
 
-stunneld.create(opts);
 }());

dist/etc/systemd/system/stunneld.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=Daplie Tunnel Server
+After=network-online.target
+Wants=network-online.target systemd-networkd-wait-online.service
+
+[Service]
+# Always restart, unless it's restarting fast enough for us to believe it's completely broken
+Restart=always
+StartLimitInterval=10
+StartLimitBurst=3
+
+User=www-data
+Group=www-data
+WorkingDirectory=/srv/stunneld
+# TODO needs --config option and these options should go in a config file
+ExecStart=/srv/stunneld/bin/stunneld.js --servernames tunnel.example.com --secret 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' --email tunnel@example.com --agree-tos
+
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+NoNewPrivileges=true
+
+[Install]
+WantedBy=multi-user.target

handlers.js

@@ -0,0 +1,118 @@
+'use strict';
+
+var http = require('http');
+var tls = require('tls');
+var wrapSocket = require('tunnel-packer').wrapSocket;
+var redirectHttps = require('redirect-https')();
+
+module.exports.create = function (program) {
+  var tunnelAdminTlsOpts = {};
+
+  // Probably a reverse proxy on an internal network (or ACME challenge)
+  function notFound(req, res) {
+    console.log('req.socket.encrypted', req.socket.encrypted);
+    res.statusCode = 404;
+    res.end("File not found.\n");
+  }
+  program.httpServer = http.createServer(
+    program.greenlock && program.greenlock.middleware(notFound)
+    || notFound
+  );
+  program.handleHttp = function (servername, socket) {
+    console.log("handleHttp('" + servername + "', socket)");
+    socket.__my_servername = servername;
+    program.httpServer.emit('connection', socket);
+  };
+
+  // Probably something that needs to be redirected to https
+  function redirectHttpsAndClose(req, res) {
+    res.setHeader('Connection', 'close');
+    redirectHttps(req, res);
+  }
+  program.httpInsecureServer = http.createServer(
+    program.greenlock && program.greenlock.middleware(redirectHttpsAndClose)
+    || redirectHttpsAndClose
+  );
+  program.handleInsecureHttp = function (servername, socket) {
+    console.log("handleInsecureHttp('" + servername + "', socket)");
+    socket.__my_servername = servername;
+    program.httpInsecureServer.emit('connection', socket);
+  };
+
+
+  //
+  // SNI is not recogonized / cannot be handled
+  //
+  program.httpInvalidSniServer = http.createServer(function (req, res) {
+    res.end("This is an old error message that shouldn't be actually be acessible anymore. If you get this please tell AJ so that he finds where it was still referenced and removes it");
+  });
+  program.tlsInvalidSniServer = tls.createServer(program.tlsOptions, function (tlsSocket) {
+    console.log('tls connection');
+    // things get a little messed up here
+    program.httpInvalidSniServer.emit('connection', tlsSocket);
+  });
+  program.httpsInvalid = function (servername, socket) {
+    // none of these methods work:
+    // httpsServer.emit('connection', socket);  // this didn't work
+    // tlsServer.emit('connection', socket);    // this didn't work either
+    //console.log('chunkLen', firstChunk.byteLength);
+
+    console.log('httpsInvalid servername', servername);
+    //program.tlsInvalidSniServer.emit('connection', wrapSocket(socket));
+    var tlsInvalidSniServer = tls.createServer(program.tlsOptions, function (tlsSocket) {
+      console.log('tls connection');
+      // things get a little messed up here
+      var httpInvalidSniServer = http.createServer(function (req, res) {
+        if (!servername) {
+          res.statusCode = 422;
+          res.end(
+            "3. An inexplicable temporal shift of the quantum realm... that makes me feel uncomfortable.\n\n"
+          + "[ERROR] No SNI header was sent. I can only think of two possible explanations for this:\n"
+          + "\t1. You really love Windows XP and you just won't let go of Internet Explorer 6\n"
+          + "\t2. You're writing a bot and you forgot to set the servername parameter\n"
+          );
+          return;
+        }
+
+        res.end(
+          "You came in hot looking for '" + servername + "' and, granted, the IP address for that domain"
+        + " must be pointing here (or else how could you be here?), nevertheless either it's not registered"
+        + " in the internal system at all (which Seth says isn't even a thing) or there is no device"
+        + " connected on the south side of the network which has informed me that it's ready to have traffic"
+        + " for that domain forwarded to it (sorry I didn't check that deeply to determine which).\n\n"
+        + "Either way, you're doing strange things that make me feel uncomfortable... Please don't touch me there any more.");
+      });
+      httpInvalidSniServer.emit('connection', tlsSocket);
+    });
+    tlsInvalidSniServer.emit('connection', wrapSocket(socket));
+  };
+
+  //
+  // To ADMIN / CONTROL PANEL of the Tunnel Server Itself
+  //
+  program.httpTunnelServer = http.createServer(function (req, res) {
+    console.log('req.socket.encrypted', req.socket.encrypted);
+    res.end('Hello, World!');
+  });
+  Object.keys(program.tlsOptions).forEach(function (key) {
+    tunnelAdminTlsOpts[key] = program.tlsOptions[key];
+  });
+  tunnelAdminTlsOpts.SNICallback = (program.greenlock && program.greenlock.httpsOptions && function (servername, cb) {
+    console.log("time to handle '" + servername + "'");
+    program.greenlock.httpsOptions.SNICallback(servername, cb);
+  }) || tunnelAdminTlsOpts.SNICallback;
+  program.tlsTunnelServer = tls.createServer(tunnelAdminTlsOpts, function (tlsSocket) {
+    console.log('tls connection');
+    // things get a little messed up here
+    (program.httpTunnelServer || program.httpServer).emit('connection', tlsSocket);
+  });
+  program.httpsTunnel = function (servername, socket) {
+    // none of these methods work:
+    // httpsServer.emit('connection', socket);  // this didn't work
+    // tlsServer.emit('connection', socket);    // this didn't work either
+    //console.log('chunkLen', firstChunk.byteLength);
+
+    console.log('httpsTunnel (Admin) servername', servername);
+    program.tlsTunnelServer.emit('connection', wrapSocket(socket));
+  };
+};

lib/device-tracker.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var Devices = module.exports;
+Devices.add = function (store, servername, newDevice) {
+  var devices = store[servername] || [];
+  devices.push(newDevice);
+  store[servername] = devices;
+};
+Devices.remove = function (store, servername, device) {
+  var devices = store[servername] || [];
+  var index = devices.indexOf(device);
+
+  if (index < 0) {
+    console.warn('attempted to remove non-present device', device.deviceId, 'from', servername);
+    return null;
+  }
+  return devices.splice(index, 1)[0];
+};
+Devices.list = function (store, servername) {
+  if (store[servername] && store[servername].length) {
+    return store[servername];
+  }
+  // There wasn't an exact match so check any of the wildcard domains, sorted longest
+  // first so the one with the biggest natural match with be found first.
+  var deviceList = [];
+  Object.keys(store).filter(function (pattern) {
+    return pattern[0] === '*' && store[pattern].length;
+  }).sort(function (a, b) {
+    return b.length - a.length;
+  }).some(function (pattern) {
+    var subPiece = pattern.slice(1);
+    if (subPiece === servername.slice(-subPiece.length)) {
+      console.log('"'+servername+'" matches "'+pattern+'"');
+      deviceList = store[pattern];
+      return true;
+    }
+  });
+
+  return deviceList;
+};
+Devices.exist = function (store, servername) {
+  return !!(Devices.list(store, servername).length);
+};
+Devices.next = function (store, servername) {
+  var devices = Devices.list(store, servername);
+  var device;
+
+  if (devices._index >= devices.length) {
+    devices._index = 0;
+  }
+  device = devices[devices._index || 0];
+  devices._index = (devices._index || 0) + 1;
+
+  return device;
+};

lib/unwrap-tls.js

@@ -0,0 +1,153 @@
+'use strict';
+
+var packer = require('tunnel-packer');
+var sni = require('sni');
+
+function pipeWs(servername, service, conn, remote) {
+  console.log('[pipeWs] servername:', servername, 'service:', service);
+
+  var browserAddr = packer.socketToAddr(conn);
+  browserAddr.service = service;
+  var cid = packer.addrToId(browserAddr);
+  conn.tunnelCid = cid;
+  console.log('[pipeWs] browser is', cid, 'home-cloud is', packer.socketToId(remote.upgradeReq.socket));
+
+  function sendWs(data, serviceOverride) {
+    if (remote.ws && (!conn.tunnelClosing || serviceOverride)) {
+      try {
+        remote.ws.send(packer.pack(browserAddr, data, serviceOverride), { binary: true });
+        // If we can't send data over the websocket as fast as this connection can send it to us
+        // (or there are a lot of connections trying to send over the same websocket) then we
+        // need to pause the connection for a little. We pause all connections if any are paused
+        // to make things more fair so a connection doesn't get stuck waiting for everyone else
+        // to finish because it got caught on the boundary. Also if serviceOverride is set it
+        // means the connection is over, so no need to pause it.
+        if (!serviceOverride && (remote.pausedConns.length || remote.ws.bufferedAmount > 1024*1024)) {
+          // console.log('pausing', cid, 'to allow web socket to catch up');
+          conn.pause();
+          remote.pausedConns.push(conn);
+        }
+      } catch (err) {
+        console.warn('[pipeWs] error sending websocket message', err);
+      }
+    }
+  }
+
+  remote.clients[cid] = conn;
+  conn.on('data', function (chunk) {
+    console.log('[pipeWs] data from browser to tunneler', chunk.byteLength);
+    sendWs(chunk);
+  });
+  conn.on('error', function (err) {
+    console.warn('[pipeWs] browser connection error', err);
+  });
+  conn.on('close', function (hadErr) {
+    console.log('[pipeWs] browser connection closing');
+    sendWs(null, hadErr ? 'error': 'end');
+    delete remote.clients[cid];
+  });
+}
+
+module.exports.createTcpConnectionHandler = function (copts) {
+  var Devices = copts.Devices;
+
+  return function onTcpConnection(conn) {
+    // this works when I put it here, but I don't know if it's tls yet here
+    // httpsServer.emit('connection', socket);
+    //tls3000.emit('connection', socket);
+
+    //var tlsSocket = new tls.TLSSocket(socket, { secureContext: tls.createSecureContext(tlsOpts) });
+    //tlsSocket.on('data', function (chunk) {
+    //  console.log('dummy', chunk.byteLength);
+    //});
+
+    //return;
+    conn.once('data', function (firstChunk) {
+      // BUG XXX: this assumes that the packet won't be chunked smaller
+      // than the 'hello' or the point of the 'Host' header.
+      // This is fairly reasonable, but there are edge cases where
+      // it does not hold (such as manual debugging with telnet)
+      // and so it should be fixed at some point in the future
+
+      // defer after return (instead of being in many places)
+      process.nextTick(function () {
+        conn.unshift(firstChunk);
+      });
+
+      var service = 'tcp';
+      var servername;
+      var str;
+      var m;
+
+      function tryTls() {
+        if (-1 !== copts.servernames.indexOf(servername)) {
+          console.log("Lock and load, admin interface time!");
+          copts.httpsTunnel(servername, conn);
+          return;
+        }
+
+        if (!servername) {
+          console.log("No SNI was given, so there's nothing we can do here");
+          copts.httpsInvalid(servername, conn);
+          return;
+        }
+
+        var nextDevice = Devices.next(copts.deviceLists, servername);
+        if (!nextDevice) {
+          console.log("No devices match the given servername");
+          copts.httpsInvalid(servername, conn);
+          return;
+        }
+
+        console.log("pipeWs(servername, service, socket, deviceLists['" + servername + "'])");
+        pipeWs(servername, service, conn, nextDevice);
+      }
+
+      // https://github.com/mscdex/httpolyglot/issues/3#issuecomment-173680155
+      if (22 === firstChunk[0]) {
+        // TLS
+        service = 'https';
+        servername = (sni(firstChunk)||'').toLowerCase();
+        console.log("tls hello servername:", servername);
+        tryTls();
+        return;
+      }
+
+      if (firstChunk[0] > 32 && firstChunk[0] < 127) {
+        str = firstChunk.toString();
+        m = str.match(/(?:^|[\r\n])Host: ([^\r\n]+)[\r\n]*/im);
+        servername = (m && m[1].toLowerCase() || '').split(':')[0];
+        console.log('servername', servername);
+        if (/HTTP\//i.test(str)) {
+          service = 'http';
+          // TODO disallow http entirely
+          // /^\/\.well-known\/acme-challenge\//.test(str)
+          if (/well-known/.test(str)) {
+            // HTTP
+            if (Devices.exist(copts.deviceLists, servername)) {
+              pipeWs(servername, service, conn, Devices.next(copts.deviceLists, servername));
+              return;
+            }
+            copts.handleHttp(servername, conn);
+          }
+          else {
+            // redirect to https
+            copts.handleInsecureHttp(servername, conn);
+          }
+          return;
+        }
+      }
+
+      console.error("Got unexpected connection", str);
+      conn.write(JSON.stringify({ error: {
+        message: "not sure what you were trying to do there..."
+      , code: 'E_INVALID_PROTOCOL' }
+      }));
+      conn.end();
+    });
+    conn.on('error', function (err) {
+      console.error('[error] tcp socket raw TODO forward and close');
+      console.error(err);
+    });
+  };
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "stunneld",
-  "version": "0.8.1",
+  "version": "0.10.0",
   "description": "A pure-JavaScript tunnel daemon for http and https similar to a localtunnel.me server, but uses TLS (SSL) with ServerName Indication (SNI) over https to work even in harsh network conditions such as in student dorms and behind HOAs, corporate firewalls, public libraries, airports, airplanes, etc. Can also tunnel tls and plain tcp.",
   "main": "wstunneld.js",
   "bin": {
@@ -13,7 +13,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/Daplie/node-tunnel-server.git"
+    "url": "git+https://git.coolaj86.com/coolaj86/tunnel-server.js.git"
   },
   "keywords": [
     "server",
@@ -42,17 +42,19 @@
   "author": "AJ ONeal <coolaj86@gmail.com> (https://coolaj86.com/)",
   "license": "(MIT OR Apache-2.0)",
   "bugs": {
-    "url": "https://github.com/Daplie/node-tunnel-server/issues"
+    "url": "https://git.coolaj86.com/coolaj86/tunnel-server.js/issues"
   },
-  "homepage": "https://github.com/Daplie/node-tunnel-server#readme",
+  "homepage": "https://git.coolaj86.com/coolaj86/tunnel-server.js",
   "dependencies": {
-    "cluster-store": "^2.0.4",
-    "commander": "^2.9.0",
-    "jsonwebtoken": "^7.1.9",
-    "localhost.daplie.com-certificates": "^1.2.3",
-    "redirect-https": "^1.1.0",
+    "bluebird": "^3.5.1",
+    "cluster-store": "^2.0.8",
+    "commander": "^2.15.1",
+    "greenlock": "^2.2.4",
+    "jsonwebtoken": "^8.2.1",
+    "localhost.daplie.me-certificates": "^1.3.5",
+    "redirect-https": "^1.1.5",
     "sni": "^1.0.0",
-    "tunnel-packer": "^1.0.0",
-    "ws": "^1.1.1"
+    "tunnel-packer": "^1.4.0",
+    "ws": "^5.1.1"
   }
 }

wstunneld.js

@@ -1,352 +1,395 @@
 'use strict';
 
-var net = require('net');
-var tls = require('tls');
-var http = require('http');
-var sni = require('sni');
 var url = require('url');
+var PromiseA = require('bluebird');
 var jwt = require('jsonwebtoken');
 var packer = require('tunnel-packer');
-var WebSocketServer = require('ws').Server;
 
+function timeoutPromise(duration) {
+  return new PromiseA(function (resolve) {
+    setTimeout(resolve, duration);
+  });
+}
+
+var Devices = require('./lib/device-tracker');
+
+module.exports.store = { Devices: Devices };
 module.exports.create = function (copts) {
+  copts.deviceLists = {};
+  //var deviceLists = {};
+  var activityTimeout = copts.activityTimeout || 2*60*1000;
+  var pongTimeout = copts.pongTimeout || 10*1000;
+  copts.Devices = Devices;
+  var onTcpConnection = require('./lib/unwrap-tls').createTcpConnectionHandler(copts);
 
-  function onWsConnection(ws) {
-		var location = url.parse(ws.upgradeReq.url, true);
-    //var token = jwt.decode(location.query.access_token);
-    var token;
+  function onWsConnection(ws, upgradeReq) {
+    console.log(ws);
+    var socketId = packer.socketToId(upgradeReq.socket);
+    var remotes = {};
 
-    try {
-      token = jwt.verify(location.query.access_token, secret);
-    } catch(e) {
-      token = null;
+    function logName() {
+      var result = Object.keys(remotes).map(function (jwtoken) {
+        return remotes[jwtoken].deviceId;
+      }).join(';');
+
+      return result || socketId;
+    }
+    function sendTunnelMsg(addr, data, service) {
+      ws.send(packer.pack(addr, data, service), {binary: true});
     }
 
-    /*
-    if (!token || !token.name) {
-      console.log('location, token');
-      console.log(location.query.access_token);
-      console.log(token);
-    }
-    */
-
-    if (!token) {
-      ws.send(JSON.stringify({ error: { message: "invalid access token", code: "E_INVALID_TOKEN" } }));
-      ws.close();
-      return;
-    }
-
-    if (!Array.isArray(token.domains)) {
-      if ('string' === typeof token.name) {
-        token.domains = [ token.name ];
-      }
-    }
-
-    if (!Array.isArray(token.domains)) {
-      ws.send(JSON.stringify({ error: { message: "invalid server name", code: "E_INVALID_NAME" } }));
-      ws.close();
-      return;
-    }
-
-    var remote;
-    token.domains.some(function (domainname) {
-      remote = remotes[domainname];
-      return remote;
-    });
-    remote = remote || {};
-    token.domains.forEach(function (domainname) {
-      console.log('domainname', domainname);
-      remotes[domainname] = remote;
-    });
-    var handlers = {
-      onmessage: function (opts) {
-        // opts.data
-        var cid = packer.addrToId(opts);
-        var cstream = remote.clients[cid];
-
-        console.log("remote '" + remote.servername + " : " + remote.id + "' has data for '" + cid + "'", opts.data.byteLength);
-
-        if (!cstream) {
-          remote.ws.send(packer.pack(opts, null, 'error'));
-          return;
-        }
-
-        cstream.browser.write(opts.data);
-      }
-    , onend: function (opts) {
-        var cid = packer.addrToId(opts);
-        console.log('[TunnelEnd]', cid);
-        handlers._onend(cid);
-      }
-    , onerror: function (opts) {
-        var cid = packer.addrToId(opts);
-        console.log('[TunnelError]', cid);
-        handlers._onend(cid);
-      }
-    , _onend: function (cid) {
-        var c = remote.clients[cid];
-        delete remote.clients[cid];
-        try {
-          c.browser.end();
-        } catch(e) {
-          // ignore
-        }
-        try {
-          c.wrapped.end();
-        } catch(e) {
-          // ignore
-        }
-      }
-    };
-    // TODO allow more than one remote per servername
-    remote.ws = ws;
-    remote.servername = token.domains.join(',');
-    remote.id = packer.socketToId(ws.upgradeReq.socket);
-    console.log("remote.id", remote.id);
-    // TODO allow tls to be decrypted by server if client is actually a browser
-    // and we haven't implemented tls in the browser yet
-    remote.decrypt = token.decrypt;
-    // TODO how to allow a child process to communicate with this one?
-    remote.clients = {};
-    remote.handle = { address: null, handle: null };
-    remote.unpacker = packer.create(handlers);
-    ws.on('message', function (chunk) {
-      console.log('message from home cloud to tunneler to browser', chunk.byteLength);
-      //console.log(chunk.toString());
-      remote.unpacker.fns.addChunk(chunk);
-    });
-    ws.on('close', function () {
-      // the remote will handle closing its local connections
-      Object.keys(remote.clients).forEach(function (cid) {
-        try {
-          remote.clients[cid].browser.end();
-        } catch(e) {
-          // ignore
+    function getBrowserConn(cid) {
+      var browserConn;
+      Object.keys(remotes).some(function (jwtoken) {
+        if (remotes[jwtoken].clients[cid]) {
+          browserConn = remotes[jwtoken].clients[cid];
+          return true;
         }
       });
-    });
-    ws.on('error', function () {
-      // ignore
-      // the remote will retry if it wants to
-    });
 
-    //store.set(token.name, remote.handle);
-	}
-
-  function connectHttp(servername, socket) {
-    console.log("connectHttp('" + servername + "', socket)");
-    socket.__my_servername = servername;
-    redirectServer.emit('connection', socket);
-  }
-
-  function connectHttps(servername, socket) {
-    // none of these methods work:
-    // httpsServer.emit('connection', socket);  // this didn't work
-    // tlsServer.emit('connection', socket);    // this didn't work either
-    //console.log('chunkLen', firstChunk.byteLength);
-
-    var myDuplex = packer.Stream.create(socket);
-
-    console.log('connectHttps servername', servername);
-    tls3000.emit('connection', myDuplex);
-
-    socket.on('data', function (chunk) {
-      console.log('[' + Date.now() + '] socket data', chunk.byteLength);
-      myDuplex.push(chunk);
-    });
-    socket.on('error', function (err) {
-      console.error('[error] connectHttps TODO close');
-      console.error(err);
-    });
-  }
-
-  function pipeWs(servername, service, browser, remote) {
-    console.log('pipeWs');
-
-    //var remote = remotes[servername];
-    var ws = remote.ws;
-    //var address = packer.socketToAddr(ws.upgradeReq.socket);
-    var baddress = packer.socketToAddr(browser);
-    var cid = packer.addrToId(baddress);
-    console.log('servername:', servername);
-    console.log('service:', service);
-    baddress.service = service;
-    var wrapForRemote = packer.Transform.create({
-      id: cid
-    //, remoteId: remote.id
-    , address: baddress
-    , servername: servername
-    , service: service
-    });
-    console.log('home-cloud is', packer.socketToId(remote.ws.upgradeReq.socket));
-    console.log('browser is', cid);
-    var bstream = remote.clients[cid] = {
-      wrapped: browser.pipe(wrapForRemote)
-    , browser: browser
-    , address: baddress
-    };
-    //var bstream = remote.clients[cid] = wrapForRemote.pipe(browser);
-    bstream.wrapped.on('data', function (pchunk) {
-      // var chunk = socket.read();
-      console.log('[bstream] data from browser to tunneler', pchunk.byteLength);
-      //console.log(JSON.stringify(pchunk.toString()));
-      try {
-        ws.send(pchunk, { binary: true });
-      } catch(e) {
-        try {
-          bstream.browser.end();
-        } catch(e) {
-          // ignore
+      return browserConn;
+    }
+    function closeBrowserConn(cid) {
+      var remote;
+      Object.keys(remotes).some(function (jwtoken) {
+        if (remotes[jwtoken].clients[cid]) {
+          remote = remotes[jwtoken];
+          return true;
         }
-      }
-    });
-    bstream.wrapped.on('error', function (err) {
-      console.error('[error] bstream.wrapped.error');
-      console.error(err);
-      try {
-        ws.send(packer.pack(baddress, null, 'error'), { binary: true });
-      } catch(e) {
-        // ignore
-      }
-      try {
-        bstream.browser.end();
-      } catch(e) {
-        // ignore
-      }
-      delete remote.clients[cid];
-    });
-    bstream.wrapped.on('end', function () {
-      try {
-        ws.send(packer.pack(baddress, null, 'end'), { binary: true });
-      } catch(e) {
-        // ignore
-      }
-      try {
-        bstream.browser.end();
-      } catch(e) {
-        // ignore
-      }
-      delete remote.clients[cid];
-    });
-  }
-
-  function onTcpConnection(browser) {
-    // this works when I put it here, but I don't know if it's tls yet here
-    // httpsServer.emit('connection', socket);
-    //tls3000.emit('connection', socket);
-
-    //var tlsSocket = new tls.TLSSocket(socket, { secureContext: tls.createSecureContext(tlsOpts) });
-    //tlsSocket.on('data', function (chunk) {
-    //  console.log('dummy', chunk.byteLength);
-    //});
-
-    //return;
-    browser.once('data', function (firstChunk) {
-      // BUG XXX: this assumes that the packet won't be chunked smaller
-      // than the 'hello' or the point of the 'Host' header.
-      // This is fairly reasonable, but there are edge cases where
-      // it does not hold (such as manual debugging with telnet)
-      // and so it should be fixed at some point in the future
-
-      // defer after return (instead of being in many places)
-      process.nextTick(function () {
-        browser.unshift(firstChunk);
       });
-
-      var service = 'tcp';
-      var servername;
-      var str;
-      var m;
-
-      function tryTls() {
-        if (!servername || (-1 !== selfnames.indexOf(servername)) || !remotes[servername]) {
-          console.log('this is a server or an unknown');
-          connectHttps(servername, browser);
-          return;
-        }
-
-        console.log("pipeWs(servername, service, socket, remotes['" + servername + "'])");
-        pipeWs(servername, service, browser, remotes[servername]);
-      }
-
-      // https://github.com/mscdex/httpolyglot/issues/3#issuecomment-173680155
-      if (22 === firstChunk[0]) {
-        // TLS
-        service = 'https';
-        servername = (sni(firstChunk)||'').toLowerCase();
-        console.log("tls hello servername:", servername);
-        tryTls();
+      if (!remote) {
         return;
       }
 
-      if (firstChunk[0] > 32 && firstChunk[0] < 127) {
-        str = firstChunk.toString();
-        m = str.match(/(?:^|[\r\n])Host: ([^\r\n]+)[\r\n]*/im);
-        servername = (m && m[1].toLowerCase() || '').split(':')[0];
-        console.log('servername', servername);
-        if (/HTTP\//i.test(str)) {
-          service = 'http';
-          if (/^\/\.well-known\/acme-challenge\//.test(str)) {
-            // HTTP
-            if (remotes[servername]) {
-              pipeWs(servername, service, browser, remotes[servername]);
-              return;
-            }
-            connectHttp(servername, browser);
-          }
-          else {
-            // redirect to https
-            connectHttp(servername, browser);
-          }
-          return;
+      PromiseA.resolve().then(function () {
+        var conn = remote.clients[cid];
+        conn.tunnelClosing = true;
+        conn.end();
+
+        // If no data is buffered for writing then we don't need to wait for it to drain.
+        if (!conn.bufferSize) {
+          return timeoutPromise(500);
+        }
+        // Otherwise we want the connection to be able to finish, but we also want to impose
+        // a time limit for it to drain, since it shouldn't have more than 1MB buffered.
+        return new PromiseA(function (resolve) {
+          var timeoutId = setTimeout(resolve, 60*1000);
+          conn.once('drain', function () {
+            clearTimeout(timeoutId);
+            setTimeout(resolve, 500);
+          });
+        });
+      }).then(function () {
+        if (remote.clients[cid]) {
+          console.warn(cid, 'browser connection still present after calling `end`');
+          remote.clients[cid].destroy();
+          return timeoutPromise(500);
+        }
+      }).then(function () {
+        if (remote.clients[cid]) {
+          console.error(cid, 'browser connection still present after calling `destroy`');
+          delete remote.clients[cid];
+        }
+      }).catch(function (err) {
+        console.warn('failed to close browser connection', cid, err);
+      });
+    }
+
+    function addToken(jwtoken) {
+      if (remotes[jwtoken]) {
+        // return { message: "token sent multiple times", code: "E_TOKEN_REPEAT" };
+        return null;
+      }
+
+      var token;
+      try {
+        token = jwt.verify(jwtoken, copts.secret);
+      } catch (e) {
+        token = null;
+      }
+
+      if (!token) {
+        return { message: "invalid access token", code: "E_INVALID_TOKEN" };
+      }
+
+      if (!Array.isArray(token.domains)) {
+        if ('string' === typeof token.name) {
+          token.domains = [ token.name ];
         }
       }
 
-      console.error("Got unexpected connection", str);
-      browser.write(JSON.stringify({ error: {
-        message: "not sure what you were trying to do there..."
-      , code: 'E_INVALID_PROTOCOL' }
-      }));
-      browser.end();
-    });
-    browser.on('error', function (err) {
-      console.error('[error] tcp socket raw TODO forward and close');
-      console.error(err);
+      if (!Array.isArray(token.domains) || !token.domains.length) {
+        return { message: "invalid server name", code: "E_INVALID_NAME" };
+      }
+      if (token.domains.some(function (name) { return typeof name !== 'string'; })) {
+        return { message: "invalid server name", code: "E_INVALID_NAME" };
+      }
+
+      // Add the custom properties we need to manage this remote, then add it to all the relevant
+      // domains and the list of all this websocket's remotes.
+      token.deviceId = (token.device && (token.device.id || token.device.hostname)) || token.domains.join(',');
+      token.ws = ws;
+      token.upgradeReq = upgradeReq;
+      token.clients = {};
+
+      token.pausedConns = [];
+      ws._socket.on('drain', function () {
+        // the websocket library has it's own buffer apart from node's socket buffer, but that one
+        // is much more difficult to watch, so we watch for the lower level buffer to drain and
+        // then check to see if the upper level buffer is still too full to write to. Note that
+        // the websocket library buffer has something to do with compression, so I'm not requiring
+        // that to be 0 before we start up again.
+        if (ws.bufferedAmount > 128*1024) {
+          return;
+        }
+
+        token.pausedConns.forEach(function (conn) {
+          if (!conn.manualPause) {
+            // console.log('resuming', conn.tunnelCid, 'now that the web socket has caught up');
+            conn.resume();
+          }
+        });
+        token.pausedConns.length = 0;
+      });
+
+      token.domains.forEach(function (domainname) {
+        console.log('domainname', domainname);
+        Devices.add(copts.deviceLists, domainname, token);
+      });
+      remotes[jwtoken] = token;
+      console.log("added token '" + token.deviceId + "' to websocket", socketId);
+      return null;
+    }
+
+    function removeToken(jwtoken) {
+      var remote = remotes[jwtoken];
+      if (!remote) {
+        return { message: 'specified token not present', code: 'E_INVALID_TOKEN'};
+      }
+
+      // Prevent any more browser connections being sent to this remote, and any existing
+      // connections from trying to send more data across the connection.
+      remote.domains.forEach(function (domainname) {
+        Devices.remove(copts.deviceLists, domainname, remote);
+      });
+      remote.ws = null;
+      remote.upgradeReq = null;
+
+      // Close all of the existing browser connections associated with this websocket connection.
+      Object.keys(remote.clients).forEach(function (cid) {
+        closeBrowserConn(cid);
+      });
+      delete remotes[jwtoken];
+      console.log("removed token '" + remote.deviceId + "' from websocket", socketId);
+      return null;
+    }
+
+    var firstToken;
+    var authn = (upgradeReq.headers.authorization||'').split(/\s+/);
+    if (authn[0] && 'basic' === authn[0].toLowerCase()) {
+      try {
+        authn = new Buffer(authn[1], 'base64').toString('ascii').split(':');
+        firstToken = authn[1];
+      } catch (err) { }
+    }
+    if (!firstToken) {
+      firstToken = url.parse(upgradeReq.url, true).query.access_token;
+    }
+    if (firstToken) {
+      var err = addToken(firstToken);
+      if (err) {
+        sendTunnelMsg(null, [0, err], 'control');
+        ws.close();
+        return;
+      }
+    }
+
+    var commandHandlers = {
+      add_token: addToken
+    , delete_token: function (token) {
+        if (token !== '*') {
+          return removeToken(token);
+        }
+        var err;
+        Object.keys(remotes).some(function (jwtoken) {
+          err = removeToken(jwtoken);
+          return err;
+        });
+        return err;
+      }
+    };
+
+    var packerHandlers = {
+      oncontrol: function (opts) {
+        var cmd, err;
+        try {
+          cmd = JSON.parse(opts.data.toString());
+        } catch (err) {}
+        if (!Array.isArray(cmd) || typeof cmd[0] !== 'number') {
+          var msg = 'received bad command "' + opts.data.toString() + '"';
+          console.warn(msg, 'from websocket', socketId);
+          sendTunnelMsg(null, [0, {message: msg, code: 'E_BAD_COMMAND'}], 'control');
+          return;
+        }
+
+        if (cmd[0] < 0) {
+          // We only ever send one command and we send it once, so we just hard coded the ID as 1.
+          if (cmd[0] === -1) {
+            if (cmd[1]) {
+              console.log('received error response to hello from', socketId, cmd[1]);
+            }
+          }
+          else {
+            console.warn('received response to unknown command', cmd, 'from', socketId);
+          }
+          return;
+        }
+
+        if (cmd[0] === 0) {
+          console.warn('received dis-associated error from', socketId, cmd[1]);
+          return;
+        }
+
+        if (commandHandlers[cmd[1]]) {
+          err = commandHandlers[cmd[1]].apply(null, cmd.slice(2));
+        }
+        else {
+          err = { message: 'unknown command "'+cmd[1]+'"', code: 'E_UNKNOWN_COMMAND' };
+        }
+
+        sendTunnelMsg(null, [-cmd[0], err], 'control');
+      }
+
+    , onmessage: function (opts) {
+        var cid = packer.addrToId(opts);
+        console.log("remote '" + logName() + "' has data for '" + cid + "'", opts.data.byteLength);
+
+        var browserConn = getBrowserConn(cid);
+        if (!browserConn) {
+          sendTunnelMsg(opts, {message: 'no matching connection', code: 'E_NO_CONN'}, 'error');
+          return;
+        }
+
+        browserConn.write(opts.data);
+        // tunnelRead is how many bytes we've read from the tunnel, and written to the browser.
+        browserConn.tunnelRead = (browserConn.tunnelRead || 0) + opts.data.byteLength;
+        // If we have more than 1MB buffered data we need to tell the other side to slow down.
+        // Once we've finished sending what we have we can tell the other side to keep going.
+        // If we've already sent the 'pause' message though don't send it again, because we're
+        // probably just dealing with data queued before our message got to them.
+        if (!browserConn.remotePaused && browserConn.bufferSize > 1024*1024) {
+          sendTunnelMsg(opts, browserConn.tunnelRead, 'pause');
+          browserConn.remotePaused = true;
+
+          browserConn.once('drain', function () {
+            sendTunnelMsg(opts, browserConn.tunnelRead, 'resume');
+            browserConn.remotePaused = false;
+          });
+        }
+      }
+
+    , onpause: function (opts) {
+        var cid = packer.addrToId(opts);
+        console.log('[TunnelPause]', cid);
+        var browserConn = getBrowserConn(cid);
+        if (browserConn) {
+          browserConn.manualPause = true;
+          browserConn.pause();
+        } else {
+          sendTunnelMsg(opts, {message: 'no matching connection', code: 'E_NO_CONN'}, 'error');
+        }
+      }
+    , onresume: function (opts) {
+        var cid = packer.addrToId(opts);
+        console.log('[TunnelResume]', cid);
+        var browserConn = getBrowserConn(cid);
+        if (browserConn) {
+          browserConn.manualPause = false;
+          browserConn.resume();
+        } else {
+          sendTunnelMsg(opts, {message: 'no matching connection', code: 'E_NO_CONN'}, 'error');
+        }
+      }
+
+    , onend: function (opts) {
+        var cid = packer.addrToId(opts);
+        console.log('[TunnelEnd]', cid);
+        closeBrowserConn(cid);
+      }
+    , onerror: function (opts) {
+        var cid = packer.addrToId(opts);
+        console.log('[TunnelError]', cid, opts.message);
+        closeBrowserConn(cid);
+      }
+    };
+    var unpacker = packer.create(packerHandlers);
+
+    var lastActivity = Date.now();
+    var timeoutId;
+    function refreshTimeout() {
+      lastActivity = Date.now();
+    }
+    function checkTimeout() {
+      // Determine how long the connection has been "silent", ie no activity.
+      var silent = Date.now() - lastActivity;
+
+      // If we have had activity within the last activityTimeout then all we need to do is
+      // call this function again at the soonest time when the connection could be timed out.
+      if (silent < activityTimeout) {
+        timeoutId = setTimeout(checkTimeout, activityTimeout-silent);
+      }
+
+      // Otherwise we check to see if the pong has also timed out, and if not we send a ping
+      // and call this function again when the pong will have timed out.
+      else if (silent < activityTimeout + pongTimeout) {
+        console.log('pinging', logName());
+        try {
+          ws.ping();
+        } catch (err) {
+          console.warn('failed to ping home cloud', logName());
+        }
+        timeoutId = setTimeout(checkTimeout, pongTimeout);
+      }
+
+      // Last case means the ping we sent before didn't get a response soon enough, so we
+      // need to close the websocket connection.
+      else {
+        console.log('home cloud', logName(), 'connection timed out');
+        ws.close(1013, 'connection timeout');
+      }
+    }
+    timeoutId = setTimeout(checkTimeout, activityTimeout);
+
+    // Note that our websocket library automatically handles pong responses on ping requests
+    // before it even emits the event.
+    ws.on('ping', refreshTimeout);
+    ws.on('pong', refreshTimeout);
+    ws.on('message', function forwardMessage(chunk) {
+      refreshTimeout();
+      console.log('message from home cloud to tunneler to browser', chunk.byteLength);
+      //console.log(chunk.toString());
+      unpacker.fns.addChunk(chunk);
     });
 
+    function hangup() {
+      clearTimeout(timeoutId);
+      console.log('home cloud', logName(), 'connection closing');
+      Object.keys(remotes).forEach(function (jwtoken) {
+        removeToken(jwtoken);
+      });
+      ws.terminate();
+    }
+
+    ws.on('close', hangup);
+    ws.on('error', hangup);
+
+    // We only ever send one command and we send it once, so we just hard code the ID as 1
+    sendTunnelMsg(null, [1, 'hello', [unpacker._version], Object.keys(commandHandlers)], 'control');
   }
 
-  var tlsOpts = copts.tlsOptions;
-  //var store = copts.store;
-
-  var remotes = {};
-  var selfnames = copts.servernames;
-  var secret = copts.secret;
-  var redirectHttps = require('redirect-https')();
-
-  var redirectServer = http.createServer(function (req, res) {
-    res.setHeader('Connection', 'close');
-    redirectHttps(req, res);
-  });
-  var httpServer = http.createServer(function (req, res) {
-    console.log('req.socket.encrypted', req.socket.encrypted);
-    res.end('Hello, World!');
-  });
-  var tls3000 = tls.createServer(tlsOpts, function (tlsSocket) {
-    console.log('tls connection');
-    // things get a little messed up here
-    httpServer.emit('connection', tlsSocket);
-  });
-  var wss = new WebSocketServer({ server: httpServer });
-
-	wss.on('connection', onWsConnection);
-
-  copts.ports.forEach(function (port) {
-    var tcp3000 = net.createServer();
-    tcp3000.listen(port, function () {
-      console.log('listening on ' + port);
-    });
-    tcp3000.on('connection', onTcpConnection);
-  });
-
+  return {
+    tcp: onTcpConnection
+  , ws: onWsConnection
+  , isClientDomain: Devices.exist.bind(null, copts.deviceLists)
+  };
 };