From 34c05fa7bc26a16069f82be05a3e17b1160e8a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Drouet?= Date: Tue, 15 Jan 2019 10:20:54 +0000 Subject: [PATCH] fix image pulling on windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jérémie Drouet --- package-lock.json | 84 +++++++++++------ package.json | 2 +- src/utils/DockerUtil.js | 199 +++++++++++++++++++++------------------- 3 files changed, 161 insertions(+), 124 deletions(-) diff --git a/package-lock.json b/package-lock.json index c1f9df0f..b3edb2b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1854,8 +1854,7 @@ "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, "buffers": { "version": "0.1.1", @@ -2110,9 +2109,7 @@ "chownr": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", - "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==", - "dev": true, - "optional": true + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==" }, "chromium-pickle-js": { "version": "0.2.0", @@ -2329,13 +2326,43 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "concat-stream": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz", - "integrity": "sha1-cIl4Yk2FavQaWnQd790mHadSwmY=", + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", "requires": { - "inherits": "~2.0.1", - "readable-stream": "~2.0.0", - "typedarray": "~0.0.5" + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + }, + "dependencies": { + "process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } } }, "configstore": { @@ -2752,12 +2779,12 @@ } }, "docker-modem": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-1.0.7.tgz", - "integrity": "sha512-PdcMwnPXgO4sN4BU+XPTjX6Ak4ZnoBwMKp+8DkDn477N/zQhk5jE1QiSAVpTn4j2TfPR5A6voVp8d5wa58iKEA==", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-1.0.8.tgz", + "integrity": "sha512-YQ2x9HUkJBxjPpppcLe34ucS9dRKkXq89dl1EZJU4DWJXkZHfjKVbOtfbi04RLC6Rgs7sfJGqS+s/ACKsOHKEw==", "requires": { "JSONStream": "1.3.2", - "debug": "^3.2.5", + "debug": "^3.2.6", "readable-stream": "~1.0.26-4", "split-ca": "^1.0.0" }, @@ -2769,7 +2796,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "requires": { "core-util-is": "~1.0.0", @@ -2781,13 +2808,13 @@ } }, "dockerode": { - "version": "2.5.4", - "resolved": "http://registry.npmjs.org/dockerode/-/dockerode-2.5.4.tgz", - "integrity": "sha512-esqrDATdckYhkOFn4BSOrqnkj3jgBkHT07uEqTRwK6na4/Rg60vjXWRopv2BbRpvFruMmKvOSNVY4MbmVBUnWw==", + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-2.5.8.tgz", + "integrity": "sha512-+7iOUYBeDTScmOmQqpUYQaE7F4vvIt6+gIZNHWhqAQEI887tiPFB9OvXI/HzQYqfUNvukMK+9myLW63oTJPZpw==", "requires": { - "concat-stream": "~1.5.1", - "docker-modem": "^1.0.0", - "tar-fs": "~1.12.0" + "concat-stream": "~1.6.2", + "docker-modem": "^1.0.8", + "tar-fs": "~1.16.3" } }, "doctrine": { @@ -9651,7 +9678,8 @@ "process-nextick-args": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" + "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=", + "dev": true }, "progress": { "version": "2.0.0", @@ -9928,6 +9956,7 @@ "version": "2.0.6", "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", + "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.1", @@ -11682,11 +11711,12 @@ } }, "tar-fs": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.12.0.tgz", - "integrity": "sha1-pqgFU9ilTHPeHQrg553ncDVgXh0=", + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", + "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", "requires": { - "mkdirp": "^0.5.0", + "chownr": "^1.0.1", + "mkdirp": "^0.5.1", "pump": "^1.0.0", "tar-stream": "^1.1.2" } diff --git a/package.json b/package.json index e9ade3aa..2489534c 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "cached-request": "1.1.2", "classnames": "2.2.5", "deep-extend": "^0.6.0", - "dockerode": "2.5.4", + "dockerode": "2.5.8", "install": "0.1.8", "jquery": "3.3.1", "mixpanel": "kitematic/mixpanel-node", diff --git a/src/utils/DockerUtil.js b/src/utils/DockerUtil.js index 027e3302..686ef53f 100644 --- a/src/utils/DockerUtil.js +++ b/src/utils/DockerUtil.js @@ -3,6 +3,7 @@ import fs from 'fs'; import path from 'path'; import dockerode from 'dockerode'; import _ from 'underscore'; +import http from 'http'; import child_process from 'child_process'; import util from './Util'; import hubUtil from './HubUtil'; @@ -13,10 +14,20 @@ import networkActions from '../actions/NetworkActions'; import networkStore from '../stores/NetworkStore'; import Promise from 'bluebird'; import rimraf from 'rimraf'; -import stream from 'stream'; -import JSONStream from 'JSONStream'; +const parseData = (item) => { + try { + return JSON.parse(item); + } catch (err) { + return null; + } +}; +const getPullingData = (raw) => + raw.split('\n') + .filter((item) => item.length > 0) + .map(parseData) + .filter((item) => !!item); var DockerUtil = { host: null, @@ -27,6 +38,7 @@ var DockerUtil = { activeContainerName: null, localImages: null, imagesUsed: [], + socketPath: util.isWindows() ? '//./pipe/docker_engine' : '/var/run/docker.sock', setup (ip, name) { if (!ip && !name) { @@ -36,11 +48,7 @@ var DockerUtil = { if (ip.indexOf('local') !== -1) { try { - if (util.isWindows()) { - this.client = new dockerode({socketPath: '//./pipe/docker_engine'}); - } else { - this.client = new dockerode({socketPath: '/var/run/docker.sock'}); - } + this.client = new dockerode({socketPath: this.socketPath}); } catch (error) { throw new Error('Cannot connect to the Docker daemon. Is the daemon running?'); } @@ -811,32 +819,25 @@ var DockerUtil = { }, pullImage (repository, tag, callback, progressCallback, blockedCallback) { - let opts = {}, config = hubUtil.config(); - if (!hubUtil.config()) { - opts = {}; - } else { + const options = { + socketPath: this.socketPath, + path: '/images/create?fromImage=' + encodeURIComponent(repository) + '&tag=' + tag, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }; + let config = hubUtil.config(); + if (hubUtil.config()) { let [username, password] = hubUtil.creds(config); - opts = { - authconfig: { - username, - password, - auth: '' - } - }; + options.headers['X-Registry-Auth'] = new Buffer(JSON.stringify({username, password})).toString('base64'); } - - this.client.pull(repository + ':' + tag, opts, (err, stream) => { - if (err) { - console.log('Err: %o', err); - callback(err); - return; - } - - stream.setEncoding('utf8'); + const req = http.request(options, (res) => { + res.setEncoding('utf8'); // scheduled to inform about progression at given interval let tick = null; - let layerProgress = {}; + const layerProgress = {}; // Split the loading in a few columns for more feedback let columns = {}; @@ -844,86 +845,92 @@ var DockerUtil = { columns.toFill = 0; // the current column index, waiting for layer IDs to be displayed let error = null; - // data is associated with one layer only (can be identified with id) - stream.pipe(JSONStream.parse()).on('data', data => { - if (data.error) { - error = data.error; - return; - } - - if (data.status && (data.status === 'Pulling dependent layers' || data.status.indexOf('already being pulled by another client') !== -1)) { - blockedCallback(); - return; - } - - if (data.status === 'Pulling fs layer') { - layerProgress[data.id] = { - current: 0, - total: 1 - }; - } else if (data.status === 'Downloading') { - if (!columns.progress) { - columns.progress = []; // layerIDs, nbLayers, maxLayers, progress value - let layersToLoad = _.keys(layerProgress).length; - let layersPerColumn = Math.floor(layersToLoad / columns.amount); - let leftOverLayers = layersToLoad % columns.amount; - for (let i = 0; i < columns.amount; i++) { - let layerAmount = layersPerColumn; - if (i < leftOverLayers) { - layerAmount += 1; - } - columns.progress[i] = {layerIDs: [], nbLayers: 0, maxLayers: layerAmount, value: 0.0}; - } + res.on('data', (rawData) => { + const items = getPullingData(rawData); + items.forEach((data) => { + if (data.error) { + error = data.error; + return; + } + + if (data.status && (data.status === 'Pulling dependent layers' || data.status.indexOf('already being pulled by another client') !== -1)) { + blockedCallback(); + return; } - layerProgress[data.id].current = data.progressDetail.current; - layerProgress[data.id].total = data.progressDetail.total; - - // Assign to a column if not done yet - if (!layerProgress[data.id].column) { - // test if we can still add layers to that column - if (columns.progress[columns.toFill].nbLayers === columns.progress[columns.toFill].maxLayers && columns.toFill < columns.amount - 1) { - columns.toFill++; - } - - layerProgress[data.id].column = columns.toFill; - columns.progress[columns.toFill].layerIDs.push(data.id); - columns.progress[columns.toFill].nbLayers++; + if (data.id && !layerProgress[data.id]) { + layerProgress[data.id] = { + current: 0, + total: 1 + }; } - if (!tick) { - tick = setTimeout(() => { - clearInterval(tick); - tick = null; + if (data.status === 'Downloading') { + if (!columns.progress) { + columns.progress = []; // layerIDs, nbLayers, maxLayers, progress value + let layersToLoad = _.keys(layerProgress).length; + let layersPerColumn = Math.floor(layersToLoad / columns.amount); + let leftOverLayers = layersToLoad % columns.amount; for (let i = 0; i < columns.amount; i++) { - columns.progress[i].value = 0.0; - if (columns.progress[i].nbLayers > 0) { - let layer; - let totalSum = 0; - let currentSum = 0; + let layerAmount = layersPerColumn; + if (i < leftOverLayers) { + layerAmount += 1; + } + columns.progress[i] = {layerIDs: [], nbLayers: 0, maxLayers: layerAmount, value: 0.0}; + } + } - for (let j = 0; j < columns.progress[i].nbLayers; j++) { - layer = layerProgress[columns.progress[i].layerIDs[j]]; - totalSum += layer.total; - currentSum += layer.current; - } + layerProgress[data.id].current = data.progressDetail.current; + layerProgress[data.id].total = data.progressDetail.total; - if (totalSum > 0) { - columns.progress[i].value = Math.min(100.0 * currentSum / totalSum, 100); - } else { - columns.progress[i].value = 0.0; + // Assign to a column if not done yet + if (!layerProgress[data.id].column) { + // test if we can still add layers to that column + if (columns.progress[columns.toFill].nbLayers === columns.progress[columns.toFill].maxLayers && columns.toFill < columns.amount - 1) { + columns.toFill++; + } + + layerProgress[data.id].column = columns.toFill; + columns.progress[columns.toFill].layerIDs.push(data.id); + columns.progress[columns.toFill].nbLayers++; + } + + if (!tick) { + tick = setTimeout(() => { + clearInterval(tick); + tick = null; + for (let i = 0; i < columns.amount; i++) { + columns.progress[i].value = 0.0; + if (columns.progress[i].nbLayers > 0) { + let layer; + let totalSum = 0; + let currentSum = 0; + + for (let j = 0; j < columns.progress[i].nbLayers; j++) { + layer = layerProgress[columns.progress[i].layerIDs[j]]; + totalSum += layer.total; + currentSum += layer.current; + } + + if (totalSum > 0) { + columns.progress[i].value = Math.min(100.0 * currentSum / totalSum, 100); + } else { + columns.progress[i].value = 0.0; + } } } - } - progressCallback(columns); - }, 16); + progressCallback(columns); + }, 16); + } } - } - }); - stream.on('end', function () { - callback(error); + }); }); + res.on('end', () => callback(error)); }); + req.on('error', (err) => { + error = err; + }); + req.end(); }, refresh () {