From 062619b63b7c1fbe1c4150707e15c20e0bd44f74 Mon Sep 17 00:00:00 2001 From: Tilman Kamp <5991088+tilmankamp@users.noreply.github.com> Date: Fri, 19 Oct 2018 18:23:37 +0200 Subject: [PATCH] WIP --- bin/prepare-job-base-image.sh | 0 bin/prepare-node.sh | 8 --- bin/prepare-service-container.sh | 36 ++++++++++++ package-lock.json | 33 +++++++++++ package.json | 1 + scripts/setup-service.sh | 60 ++++++++++++++++++++ {bin => scripts}/snakepit.conf.template | 0 snakepit.service => scripts/snakepit.service | 5 +- src/config.js | 8 ++- src/jobs.js | 20 ++++--- src/service.js | 2 + src/store.js | 2 +- 12 files changed, 152 insertions(+), 23 deletions(-) create mode 100644 bin/prepare-job-base-image.sh delete mode 100644 bin/prepare-node.sh create mode 100755 bin/prepare-service-container.sh create mode 100755 scripts/setup-service.sh rename {bin => scripts}/snakepit.conf.template (100%) rename snakepit.service => scripts/snakepit.service (67%) diff --git a/bin/prepare-job-base-image.sh b/bin/prepare-job-base-image.sh new file mode 100644 index 0000000..e69de29 diff --git a/bin/prepare-node.sh b/bin/prepare-node.sh deleted file mode 100644 index 5b5319b..0000000 --- a/bin/prepare-node.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -apt update -apt install nfs-common openssh-server - -adduser --disabled-password --gecos "" -u 1555 snakepit -adduser --disabled-password --gecos "" -u 2555 snake -echo 'snakepit ALL=(snake) NOPASSWD: /bin/bash' | sudo EDITOR='tee -a' visudo diff --git a/bin/prepare-service-container.sh b/bin/prepare-service-container.sh new file mode 100755 index 0000000..e3175e0 --- /dev/null +++ b/bin/prepare-service-container.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -e +if [ $# -ne 1 ] && [ $# -ne 2 ] ; then + echo "Usage: prepare-service-container.sh [code-path]" + exit 1 +fi + +uid=`ls -ldn "$1" | awk '{print $3}'` + +if lxc network show snakebr0 > /dev/null 2>&1; then + address=`lxc network get snakebr0 ipv4.address` +else + if ! lxc network show lxdbr0 > /dev/null 2>&1; then + lxc network create lxdbr0 + fi + address=`lxc network get lxdbr0 ipv4.address` +fi +address="`echo "$address" | cut -d/ -f 1`" + +lxc init ubuntu:18.04 snakepit +lxc config set snakepit raw.idmap "both $uid 0" +lxc config device add snakepit data disk path=/data source="$1" +if [ $# -eq 2 ]; then + lxc config device add snakepit code disk path=/code source="$2" +fi + +lxc start snakepit +exe="lxc exec snakepit -- " +$exe systemctl isolate multi-user.target +$exe apt update +$exe apt install -y curl jq nodejs npm git build-essential +if [ $# -ne 2 ]; then + $exe bash -c 'git clone https://github.com/mozilla/snakepit.git /code; cd /code; npm install' +fi + +$exe /code/scripts/setup-service.sh "https://${address}:8443/1.0" diff --git a/package-lock.json b/package-lock.json index 6f7957c..7a71174 100644 --- a/package-lock.json +++ b/package-lock.json @@ -388,6 +388,11 @@ } } }, + "dset": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/dset/-/dset-2.0.1.tgz", + "integrity": "sha512-nI29OZMRYq36hOcifB6HTjajNAAiBKSXsyWZrq+VniusseuP2OpNlTiYgsaNRSGvpyq5Wjbc2gQLyBdTyWqhnQ==" + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -911,6 +916,31 @@ "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" }, + "lxdn": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/lxdn/-/lxdn-0.1.6.tgz", + "integrity": "sha512-KTDcsO//PHxAjrFL1jTzNqcEPy4bLeFG3w5+1yig7oGa/AfBvRBh074kVBlQzyz0DFTgoNq5dYkGJo52FfJF5w==", + "requires": { + "debug": "4.1.0", + "dset": "2.0.1", + "node-fetch": "github:mkg20001/node-fetch#6bea5c1fc1c22230c87202102518df4fbe980e1e" + }, + "dependencies": { + "debug": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.0.tgz", + "integrity": "sha512-heNPJUJIqC+xB6ayLAMHaIrmN9HKa7aQO8MGqKpvCA+uJYVcvR6l5kgdrhRuwPFHU7P5/A1w0BjByPHwpfTDKg==", + "requires": { + "ms": "2.1.1" + } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + } + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -1054,6 +1084,9 @@ "minimatch": "3.0.4" } }, + "node-fetch": { + "version": "github:mkg20001/node-fetch#6bea5c1fc1c22230c87202102518df4fbe980e1e" + }, "node-pre-gyp": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.9.1.tgz", diff --git a/package.json b/package.json index f6d832a..1b84132 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "httpfslib": "https://github.com/mozilla/httpfslib.git", "js-yaml": "^3.12.0", "jsonwebtoken": "^8.2.2", + "lxdn": "^0.1.6", "morgan": "^1.9.0", "multer": "^1.3.0", "multi-integer-range": "^4.0.4", diff --git a/scripts/setup-service.sh b/scripts/setup-service.sh new file mode 100755 index 0000000..08985a6 --- /dev/null +++ b/scripts/setup-service.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -e + +lxd_endpoint=$1 + +command=$'\n "sudo lxc exec snakepit -- /code/scripts/setup-service.sh '$lxd_endpoint$'"\n' +if ! curl -k -s $lxd_endpoint > /dev/null 2>&1; then + echo "Problem accessing \"$lxd_endpoint\". Please call $command with the appropriate LXD REST service endpoint." + exit 1 +fi + +config_dir="/etc/snakepit" +mkdir -p "$config_dir" + +openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \ + -subj "/C=US/ST=California/L=San Francisco/O=Mozilla/CN=mozilla.com" \ + -keyout "$config_dir/lxd.key" \ + -out "$config_dir/lxd.crt" \ + > /dev/null 2>&1 +echo -n "Local LXD trust password: " +read -s password +echo "" +json=$(curl \ + -s \ + -k \ + --cert "$config_dir/lxd.crt" \ + --key "$config_dir/lxd.key" \ + $lxd_endpoint/certificates \ + -X POST \ + -d "{\"type\": \"client\", \"password\": \"$password\"}" \ +) +if [[ "`echo $json | jq '.status_code'`" -ne "200" ]]; then + echo "Problem authenticating at \"$lxd_endpoint\". Please call $command again and provide the correct password." + exit 2 +fi + +token_secret_path="$config_dir/token-secret.txt" +touch "$token_secret_path" +chmod 600 "$token_secret_path" +openssl rand -base64 32 >"$token_secret_path" + +conf="$config_dir/snakepit.conf" +touch "$conf" +chmod 644 "$conf" +cp "/code/scripts/snakepit.conf.template" "$conf" +echo -e "# Local interface the snakepit service should use (e.g. 192.168.1.1).\ninterface: 0.0.0.0\n" >>$conf +echo -e "# Local port of the snakepit service.\nport: 80\n" >>$conf +echo -e "# External URL.\nexternal: \"http://snakepit.lxd\"\n" >>$conf +echo -e "# LXD REST URL.\nlxd: \"$lxd_endpoint\"\n" >>$conf +echo -e "# LXD REST certificate.\nlxdcert: \"$config_dir/lxd.crt\"\n" >>$conf +echo -e "# LXD REST key.\nlxdkey: \"$config_dir/lxd.key\"\n" >>$conf +echo -e "# Path to data root.\ndataRoot: \"/data\"\n" >>$conf +echo -e "# Path to session token secret file.\ntokenSecretPath: \"$token_secret_path\"\n" >>$conf + +if systemctl is-active --quiet snakepit; then + systemctl stop snakepit +fi +cp /code/scripts/snakepit.service /lib/systemd/system/ +systemctl enable snakepit +systemctl start snakepit diff --git a/bin/snakepit.conf.template b/scripts/snakepit.conf.template similarity index 100% rename from bin/snakepit.conf.template rename to scripts/snakepit.conf.template diff --git a/snakepit.service b/scripts/snakepit.service similarity index 67% rename from snakepit.service rename to scripts/snakepit.service index 59f0879..592d804 100644 --- a/snakepit.service +++ b/scripts/snakepit.service @@ -5,10 +5,9 @@ After=network.target [Service] Environment=NODE_ENV=production -WorkingDirectory=/home/snakepit +WorkingDirectory=/code Type=simple -User=snakepit -ExecStart=/usr/local/bin/snakepit +ExecStart=/usr/bin/node /code/src/service.js Restart=on-failure [Install] diff --git a/src/config.js b/src/config.js index af33e67..9d7cad0 100644 --- a/src/config.js +++ b/src/config.js @@ -8,9 +8,13 @@ const oneMinute = 60 * oneSecond const oneHour = 60 * oneMinute const oneDay = 24 * oneHour -var filename = process.env.SNAKEPIT_CONF || '/etc/snakepit.conf' +var filename = process.env.SNAKEPIT_CONF || '/etc/snakepit/snakepit.conf' if (!fs.existsSync(filename)) { - filename = path.join(process.env.HOME, '.snakepit', 'snakepit.conf') + if (process.env.HOME) { + filename = path.join(process.env.HOME, '.snakepit', 'snakepit.conf') + } else { + filename = path.join('config', 'snakepit.conf') + } } function tryConfigFile(fun, verb) { diff --git a/src/jobs.js b/src/jobs.js index f080bfc..c40f21f 100644 --- a/src/jobs.js +++ b/src/jobs.js @@ -326,16 +326,18 @@ function _createJobDescription(dbjob) { } } return { - id: dbjob.id, - description: dbjob.description, - user: dbjob.user, - groups: dbjob.groups, - resources: dbjob.state >= jobStates.STARTING ? summarizeClusterReservation(dbjob.clusterReservation, true) : dbjob.clusterRequest, - state: dbjob.state, - since: duration, + id: dbjob.id, + description: dbjob.description, + user: dbjob.user, + groups: dbjob.groups, + resources: dbjob.state >= jobStates.STARTING ? + summarizeClusterReservation(dbjob.clusterReservation, true) : + dbjob.clusterRequest, + state: dbjob.state, + since: duration, schedulePosition: db.schedule.indexOf(dbjob.id), - utilComp: utilComp / utilCompCount, - utilMem: utilMem / utilMemCount + utilComp: utilComp / utilCompCount, + utilMem: utilMem / utilMemCount } } diff --git a/src/service.js b/src/service.js index ab57356..2427e74 100644 --- a/src/service.js +++ b/src/service.js @@ -1,10 +1,12 @@ const cluster = require('cluster') const cpus = require('os').cpus().length +const lxdn = require('lxdn') const config = require('./config.js') const modules = 'jobfs users groups nodes jobs aliases' .split(' ').map(name => require('./' + name + '.js')) if (cluster.isMaster) { + let mainClient = lxdn(config.lxd, { key: config.lxdkey, cert: config.lxdcert }) modules.forEach(module => (module.initDb || Function)()) modules.forEach(module => (module.tick || Function)()) for (let i = 0; i < cpus; i++) { diff --git a/src/store.js b/src/store.js index c20050c..82e5349 100644 --- a/src/store.js +++ b/src/store.js @@ -155,7 +155,7 @@ function _writeDb() { storeLog = [] fs.writeFile(DB_PATH, JSON.stringify(rawRoot, null, '\t'), function(err) { if(err) - return console.err(err); + return console.error(err); //log('Wrote db!') }) }