From 01b5e0fb3caad166c456dae2d7f5f38823cda393 Mon Sep 17 00:00:00 2001 From: Tilman Kamp <5991088+tilmankamp@users.noreply.github.com> Date: Fri, 10 May 2019 15:44:45 +0200 Subject: [PATCH] Port forwarding --- package-lock.json | 129 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 2 + src/pit.js | 93 ++++++++++++++++++++++++++++----- 3 files changed, 208 insertions(+), 16 deletions(-) diff --git a/package-lock.json b/package-lock.json index ffad7f2..9e11ad4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -136,6 +136,17 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" }, + "duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "requires": { + "end-of-stream": "1.4.1", + "inherits": "2.0.3", + "readable-stream": "2.3.6", + "stream-shift": "1.0.0" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -145,6 +156,14 @@ "safer-buffer": "2.1.2" } }, + "end-of-stream": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", + "requires": { + "once": "1.4.0" + } + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -251,11 +270,21 @@ "ms": "2.1.1" } }, + "inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", @@ -323,6 +352,18 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, + "multiplex": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/multiplex/-/multiplex-6.7.0.tgz", + "integrity": "sha1-/3Pk5AB5FwxEQtFgllZY+N75YMI=", + "requires": { + "duplexify": "3.7.1", + "inherits": "2.0.3", + "readable-stream": "2.3.6", + "varint": "4.0.1", + "xtend": "4.0.1" + } + }, "nan": { "version": "2.11.1", "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.1.tgz", @@ -340,6 +381,14 @@ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1.0.2" + } + }, "os-tmpdir": { "version": "1.0.2", "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -350,6 +399,11 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -370,6 +424,20 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "1.0.2", + "inherits": "2.0.3", + "isarray": "1.0.0", + "process-nextick-args": "2.0.0", + "safe-buffer": "5.1.2", + "string_decoder": "1.1.1", + "util-deprecate": "1.0.2" + } + }, "readline-sync": { "version": "1.4.9", "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.9.tgz", @@ -428,6 +496,19 @@ "tweetnacl": "0.14.5" } }, + "stream-shift": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", + "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "5.1.2" + } + }, "tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -465,6 +546,11 @@ "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" }, + "ultron": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", + "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==" + }, "uri-js": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", @@ -473,11 +559,21 @@ "punycode": "2.1.1" } }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" }, + "varint": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/varint/-/varint-4.0.1.tgz", + "integrity": "sha1-SQgpuULSSEY7KzUJeZXDv3NxmOk=" + }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", @@ -488,6 +584,36 @@ "extsprintf": "1.3.0" } }, + "websocket-stream": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/websocket-stream/-/websocket-stream-5.5.0.tgz", + "integrity": "sha512-EXy/zXb9kNHI07TIMz1oIUIrPZxQRA8aeJ5XYg5ihV8K4kD1DuA+FY6R96HfdIHzlSzS8HiISAfrm+vVQkZBug==", + "requires": { + "duplexify": "3.7.1", + "inherits": "2.0.3", + "readable-stream": "2.3.6", + "safe-buffer": "5.1.2", + "ws": "3.3.3", + "xtend": "4.0.1" + }, + "dependencies": { + "ws": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz", + "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==", + "requires": { + "async-limiter": "1.0.0", + "safe-buffer": "5.1.2", + "ultron": "1.1.1" + } + } + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, "ws": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.0.tgz", @@ -499,8 +625,7 @@ "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=", - "optional": true + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" } } } diff --git a/package.json b/package.json index 0fa78c6..8a55f80 100644 --- a/package.json +++ b/package.json @@ -40,10 +40,12 @@ "hoek": ">=6.1.2", "matcher": "^1.1.1", "mime": ">=2.4.0", + "multiplex": "^6.7.0", "progress": "^2.0.3", "readline-sync": "^1.4.9", "request": "^2.88.0", "tmp": "^0.0.33", + "websocket-stream": "^5.5.0", "ws": "^6.2.0" }, "optionalDependencies": { diff --git a/src/pit.js b/src/pit.js index 464d866..7ef41b6 100755 --- a/src/pit.js +++ b/src/pit.js @@ -1,14 +1,17 @@ #! /usr/bin/env node const fs = require('fs') const os = require('os') +const net = require('net') const url = require('url') const tmp = require('tmp') const path = require('path') -const program = require('commander') const WebSocket = require('ws') +const websocket = require('websocket-stream') const request = require('request') -const ProgressBar = require('progress') const filesize = require('filesize') +const program = require('commander') +const multiplex = require('multiplex') +const ProgressBar = require('progress') const readlineSync = require('readline-sync') const { spawn, execFileSync } = require('child_process') @@ -399,6 +402,10 @@ function printEntityHelp() { printLine('Accepted values for "entity": ' + Array.prototype.slice.call(arguments).join(', ') + '.') } +function printJobNumberHelp() { + printLine('"jobNumber": Number of the targeted job') +} + function printPropertyHelp() { printLine('Properties are pairs of property-name and value of the form "property=value".') } @@ -664,6 +671,16 @@ function copyContent (entity, remotePath, localPath, options) { }) } +function toWebSocketUrl(httpurl) { + let endpoint = url.parse(httpurl) + if (endpoint.protocol == 'https:') { + endpoint.protocol = 'wss' + } else { + endpoint.protocol = 'ws' + } + return url.format(endpoint) +} + program .version('0.0.1') @@ -931,6 +948,8 @@ program .on('--help', function() { printIntro() printExample('pit stop 1234') + printLine() + printJobNumberHelp() }) .action(function(jobNumber) { callPit('post', 'jobs/' + jobNumber + '/stop', evaluateResponse) @@ -1011,15 +1030,13 @@ program program .command('log ') .description('show job\'s log') - .option('-f, --follow', 'continuously shows further log output if the job is still running') .on('--help', function() { printIntro() - printExample('pit log -f') + printExample('pit log 1234') printLine() + printJobNumberHelp() }) - .action((jobNumber, options) => { - showLog(jobNumber) - }) + .action(jobNumber => showLog(jobNumber)) program .command('exec -- ...') @@ -1032,17 +1049,12 @@ program printExample('pit exec 1234 -- ls -la /') printExample('pit exec -w 1 1234 -- cat /data/rw/pit/src/.compute >1234.compute') printLine() + printJobNumberHelp() }) .action((jobNumber, options) => { let instance = '' + (options.worker || 0) getConnectionSettings(connection => { - let endpoint = url.parse(connection.url) - if (endpoint.protocol == 'https:') { - endpoint.protocol = 'wss' - } else { - endpoint.protocol = 'ws' - } - endpoint = url.format(endpoint) + let endpoint = toWebSocketUrl(connection.url) let stdin = process.stdin let stdout = process.stdout let stderr = process.stderr @@ -1108,6 +1120,59 @@ program }) }) +program + .command('forward [ports...]') + .description('forward ports of a job\'s worker to localhost') + .option('-w, --worker ', 'index of the target worker (defaults to 0)') + .on('--help', function() { + printIntro() + printExample('pit forward 1234 8080:80 7022:22') + printExample('pit forward 1234 8080') + printLine() + printJobNumberHelp() + printLine('"ports": All the ports to forward. Each port has to be provided either as one number (local and remote port being the same) or as a colon-separated pair where the first one is the local and the second one the remote counter-part.') + }) + .action((jobNumber, ports, options) => { + let instance = '' + (options.worker || 0) + let portPairs = {} + for (let port of ports) { + let [localPort, remotePort] = port.split(':').map(x => Number(x)) + remotePort = remotePort || localPort + if (!localPort) { + fail('Wrong port pair format') + } + portPairs[localPort] = remotePort + } + getConnectionSettings(connection => { + let endpoint = toWebSocketUrl(connection.url) + let ws = websocket(endpoint + 'jobs/' + jobNumber + '/instances/' + instance + '/forward', { + headers: { 'X-Auth-Token': connection.token }, + ca: connection.ca + }) + let mp = multiplex() + mp.pipe(ws) + ws.pipe(mp) + let idc = 0 + let onConnection = socket => { + let remotePort = portPairs[socket.localPort] + let id = idc++ + let stream = mp.createStream(id + '-' + remotePort) + socket.pipe(stream) + stream.pipe(socket) + stream.on('error', err => { console.error('Remote', err.message || 'problem'); socket.end() }) + } + for (let localPort of Object.keys(portPairs)) { + let remotePort = portPairs[localPort] + console.log('Forwarding port ' + remotePort + ' of worker ' + instance + ' to port ' + localPort + ' on localhost...') + let server = net.createServer(onConnection) + server.listen(localPort, 'localhost') + } + console.log('Hit Ctrl-C to stop forwarding.') + mp.on('error', err => fail('Problem with remote end - Closing')) + ws.on('error', err => fail('Problem opening connection to pit: ' + err)) + }) + }) + program .command('ls [remotePath]') .description('lists contents within a job directory')