From 5ff97d1f751d9f75d128fc1683a2e6a87e290aa4 Mon Sep 17 00:00:00 2001 From: Zachary Carter Date: Fri, 16 Nov 2012 13:34:08 -0800 Subject: [PATCH] WIP build server --- .awsbox.json | 3 +- bin/builder | 277 +++++++++++++++++++++++++++++++++++++++++ bin/router | 32 ++++- etc/config.js | 5 + package.json | 3 +- scripts/crxmake.sh | 44 +++++++ scripts/run_locally.js | 18 +-- 7 files changed, 361 insertions(+), 21 deletions(-) create mode 100644 bin/builder create mode 100755 scripts/crxmake.sh diff --git a/.awsbox.json b/.awsbox.json index cee8088..912f942 100644 --- a/.awsbox.json +++ b/.awsbox.json @@ -2,7 +2,8 @@ "processes": [ "bin/router", "bin/api", - "bin/static" + "bin/static", + "bin/builder" ], "hooks": { "postcreate": "scripts/post_create.sh" diff --git a/bin/builder b/bin/builder new file mode 100644 index 0000000..0455dbd --- /dev/null +++ b/bin/builder @@ -0,0 +1,277 @@ +#!/usr/bin/env node + +const temp = require('temp'); +const path = require('path'); +const util = require('util'); +const events = require('events'); +const http = require('http'); +const git = require('awsbox/lib/git.js'); +const fs = require('fs'); +const express = require('express'); +const irc = require('irc'); +const config = require('../etc/config'); +const spawn = require('child_process').spawn; + +console.log("build server starting up"); + +var buildScript = path.resolve(__dirname, '..', 'script', 'crxmake.sh'); +var downloadDir = path.join(__dirname, '..', 'downloads'); + +var githubRepo = 'git://github.com/mozilla/skycrane'; +var devBranch = 'master'; +var downloadFile = 'latest.crx'; + +var ircChannel = '#tobmog'; + +// a class capable of deploying and emmitting events along the way +function Builder(options) { + events.EventEmitter.call(this); + if (!options) optiosn = {}; + + this.repo = options.repo; + this.branch = options.branch; + + // a directory where we'll keep code + this._codeDir = process.env['CODE_DIR'] || temp.mkdirSync(); + console.log("code dir is:", this._codeDir); + var self = this; + + git.init(this._codeDir, function(err) { + if (err) { + console.log("can't init code dir:", err); + process.exit(1); + } + self.emit('ready'); + }); +} + +util.inherits(Builder, events.EventEmitter); + +Builder.prototype._writeSha = function(sha, cb) { + var self = this; + fs.writeFile(path.join(downloadDir, 'ver.txt'), sha, function(err, sha) { + if (err) self.emit('info', 'could not write last sha'); + if (cb) cb(err); + }); +}; + +Builder.prototype._getLatestRunningSHA = function(cb) { + var self = this; + fs.readFile(path.join(downloadDir, 'ver.txt'), 'utf8', function(err, sha) { + if (err) self.emit('info', 'could not get last sha'); + if (cb) cb(err, sha); + }); +}; + +Builder.prototype._buildNewCode = function(cb) { + var self = this; + + function splitAndEmit(chunk) { + if (chunk) chunk = chunk.toString(); + if (typeof chunk === 'string') { + chunk.split('\n').forEach(function (line) { + line = line.trim(); + if (line.length) self.emit('progress', line); + }); + } + } + + console.log(buildScript); + var crxBuild = spawn(buildScript, [ self._codeDir, '~/.ssh/id_rsa' ], { cwd: self._codeDir }); + + crxBuild.stdout.on('data', splitAndEmit); + crxBuild.stderr.on('data', splitAndEmit); + + crxBuild.on('exit', function(code, signal) { + if (code != 0) { + cb('could not build crx'); + return; + } + }); +}; + +Builder.prototype._pullLatest = function(cb) { + var self = this; + git.pull(this._codeDir, this.repo, this.branch, function(l) { + self.emit('progress', l); + }, function(err) { + if (err) return cb(err); + git.currentSHA(self._codeDir, function(err, latest) { + if (err) return cb(err); + self.emit('info', 'latest available sha is ' + latest); + self._getLatestRunningSHA(function(err, running) { + if (latest !== running) { + self.emit('deployment_begins', { + sha: latest, + }); + var startTime = new Date(); + + self._buildNewCode(function(err, res) { + if (err) return cb(err); + self._writeSha(latest, function(err) { + // deployment is complete! + self.emit('deployment_complete', { + sha: latest, + time: (new Date() - startTime) + }); + cb(null, null); + }); + }); + } else { + self.emit('info', 'up to date'); + cb(null, null); + } + }); + }); + }); +} + +// may be invoked any time we suspect updates have occured to re-deploy +// if needed +Builder.prototype.checkForUpdates = function() { + var self = this; + + if (this._busy) { + self.emit('info', 'busy'); + return; + } + + this._busy = true; + self.emit('info', 'checking for updates'); + + self._pullLatest(function(err, sha) { + if (err) self.emit('error', err); + self.emit('info', 'done checking'); + self._busy = false; + }); +} + +// create dev builder +var builder = new Builder({ repo: githubRepo, branch: devBranch }); + +var currentLogFile = null; +// a directory where we'll keep deployment logs +var deployLogDir = process.env['DEPLOY_LOG_DIR'] || temp.mkdirSync(); + +var deployingSHA = null; + +console.log("deployment log dir is:", deployLogDir); + +[ 'info', 'ready', 'error', 'deployment_begins', 'deployment_complete', 'progress' ].forEach(function(evName) { + builder.on(evName, function(data) { + if (data !== null && data !== undefined && typeof data != 'string') data = JSON.stringify(data, null, 2); + var msg = evName + (data ? (": " + data) : "") + console.log(msg) + if (currentLogFile) currentLogFile.write(msg + "\n"); + }); +}); + +// irc integration! +var ircClient = null; +function ircSend(msg) { + if (!ircClient) { + ircClient = new irc.Client('irc.mozilla.org', 'gombot_builder', { + channels: [ircChannel] + }); + ircClient.on('error', function(e) { + console.log('irc error: ', e); + }); + ircClient.once('join' + ircChannel, function(e) { + ircClient.say(ircChannel, msg); + }); + } else { + ircClient.say(ircChannel, msg); + } +} + +function ircDisconnect() { + setTimeout(function() { + if (ircClient) { + ircClient.disconnect(); + ircClient = null; + } + }, 1000); +} + + +// now when deployment begins, we log all events +builder.on('deployment_begins', function(r) { + currentLogFile = fs.createWriteStream(path.join(deployLogDir, r.sha + ".txt")); + currentLogFile.write("deployment of " + r.sha + " begins\n"); + deployingSHA = r.sha; + ircSend("deploying " + r.sha); +}); + +function closeLogFile() { + if (currentLogFile) { + currentLogFile.end(); + currentLogFile = null; + } +} + +builder.on('deployment_complete', function(r) { + ircSend("deployment of " + deployingSHA + " completed successfully in " + + (r.time / 1000.0).toFixed(2) + "s"); + ircDisconnect(); + + closeLogFile(); + deployingSHA = null; + + // always check to see if we should try another deployment after one succeeds to handle + // rapid fire commits + console.log('from complete'); + builder.checkForUpdates(); +}); + +builder.on('error', function(r) { + ircSend("deployment of " + deployingSHA + " failed. check logs for deets"); + ircDisconnect(); + + closeLogFile(); + deployingSHA = null; + + // on error, try again in 2 minutes + setTimeout(function () { + console.log('from error'); + builder.checkForUpdates(); + }, 2 * 60 * 1000); +}); + + +// We check every 15 minutes, in case a cosmic ray hits and github's +// webhooks fail, or other unexpected errors occur +setInterval(function () { + console.log('from interval'); + builder.checkForUpdates(); +}, (1000 * 60 * 15)); + +// check for updates at startup +builder.on('ready', function() { + console.log('from ready'); + builder.checkForUpdates(); +}); + +// setup build server +var app = express(); +var server = http.createServer(app); + +var check = function(req, res) { + console.log('from check'); + builder.checkForUpdates(); + res.send('ok'); +}; + +app.get('/check', check); +app.post('/check', check); + +app.get('/', function(req, res) { + var what = "idle"; + if (deployingSHA) what = "deploying " + deployingSHA; + res.send(what); +}); + +app.use(express.static(deployLogDir)); + +server.listen(config.process.builder.port, config.process.builder.host, function() { + console.log("running on http://" + server.address().address + ":" + server.address().port); +}); diff --git a/bin/router b/bin/router index cb6bb53..bdb5d6b 100755 --- a/bin/router +++ b/bin/router @@ -12,8 +12,9 @@ const forward = require('../lib/http_forward').forward; var app = express(); var server = http.createServer(app); -var api_url = config.api_url; -var static_url = config.static_url; +var api_url = config.api_url; +var static_url = config.static_url; +var builder_url = config.builder_url + '/check'; console.log("router starting up"); @@ -23,8 +24,28 @@ app.use(express.logger()); // redirect requests to the "verifier" processes app.use(function(req, res, next) { if (/^\/api/.test(req.url)) { - forward( - api_url+req.url.replace(/^\/api/,''), req, res, + forward(api_url + req.url.replace(/^\/api/,''), + req, res, + function(err) { + if (err) { + console.error("error forwarding request:", err); + req.destroy(); + } + }); + } else { + return next(); + } +}); + +// kick-off addon updates +app.use(function(req, res, next) { + if (req.url === '/update_addons') { + // grab addon code from github + // run extension packager script + // issue update to installed extensions + // - dev: updated every push + // - nightly: updated every day to last dev + forward(builder_url, req, res, function(err) { if (err) { console.error("error forwarding request:", err); @@ -38,8 +59,7 @@ app.use(function(req, res, next) { //static catch-all app.use(function(req, res, next) { - forward( - static_url+req.url, req, res, + forward(static_url + req.url, req, res, function(err) { if (err) { console.error("error forwarding request:", err); diff --git a/etc/config.js b/etc/config.js index 5065d56..80b1c3a 100644 --- a/etc/config.js +++ b/etc/config.js @@ -13,6 +13,10 @@ var config = module.exports = { static: { port: 20002, host: '127.0.0.1' + }, + builder: { + port: 20003, + host: '127.0.0.1' } }, hapi: { @@ -29,3 +33,4 @@ var config = module.exports = { config.api_url = 'http://' + config.process.api.host + ':' + config.process.api.port; config.static_url = 'http://' + config.process.static.host + ':' + config.process.static.port; +config.builder_url = 'http://' + config.process.builder.host + ':' + config.process.builder.port; diff --git a/package.json b/package.json index 86f515c..1d226cc 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "hapi": "git://github.com/lloyd/hapi#7e6cf0", "walkdir": "0.0.5", "express": "3.0.2", - "nunjucks": "0.1.5" + "nunjucks": "0.1.5", + "irc": "0.3.3" }, "optionalDependencies": { "couchbase": "0.0.4" diff --git a/scripts/crxmake.sh b/scripts/crxmake.sh new file mode 100755 index 0000000..0f86495 --- /dev/null +++ b/scripts/crxmake.sh @@ -0,0 +1,44 @@ +#!/bin/bash -e +# +# Purpose: Pack a Chromium extension directory into crx format + +if test $# -ne 2; then + echo "Usage: crxmake.sh " + exit 1 +fi + +dir=$1 +key=$2 || ~/.ssh/id_rsa +name=$(basename "$dir") +crx="$name.crx" +pub="$name.pub" +sig="$name.sig" +zip="$name.zip" +trap 'rm -f "$pub" "$sig" "$zip"' EXIT + +echo "oh hai" + +# zip up the crx dir +cwd=$(pwd -P) +(cd "$dir" && zip -qr -9 -X "$cwd/$zip" .) + +# signature +openssl sha1 -sha1 -binary -sign "$key" < "$zip" > "$sig" + +# public key +openssl rsa -pubout -outform DER < "$key" > "$pub" 2>/dev/null + +byte_swap () { + # Take "abcdefgh" and return it as "ghefcdab" + echo "${1:6:2}${1:4:2}${1:2:2}${1:0:2}" +} + +crmagic_hex="4372 3234" # Cr24 +version_hex="0200 0000" # 2 +pub_len_hex=$(byte_swap $(printf '%08x\n' $(ls -l "$pub" | awk '{print $5}'))) +sig_len_hex=$(byte_swap $(printf '%08x\n' $(ls -l "$sig" | awk '{print $5}'))) +( + echo "$crmagic_hex $version_hex $pub_len_hex $sig_len_hex" | xxd -r -p + cat "$pub" "$sig" "$zip" +) > "$crx" +echo "Wrote $crx" diff --git a/scripts/run_locally.js b/scripts/run_locally.js index 26cfaf6..88b188a 100755 --- a/scripts/run_locally.js +++ b/scripts/run_locally.js @@ -13,9 +13,13 @@ const HOST = process.env['GOMBOT_IP_ADDRESS'] || process.env['GOMBOT_HOST'] || " var daemonsToRun = { api: { }, static: { }, + builder: { }, router: { } }; +// only run builder if specified +//if (!process.env.BUILD_SERVER) delete daemonsToRun.builder; + process.env['GOMBOT_HOST'] = HOST; // use the "local" configuration @@ -32,22 +36,10 @@ process.env['LOG_TO_CONSOLE'] = 1; process.env['GOMBOT_ROUTER_URL'] = 'http://' + HOST + ":20000"; process.env['GOMBOT_API_URL'] = 'http://' + HOST + ":20001"; process.env['GOMBOT_STATIC_URL'] = 'http://' + HOST + ":20002"; +process.env['GOMBOT_BUILDER_URL'] = 'http://' + HOST + ":20003"; process.env['PUBLIC_URL'] = process.env['GOMBOT_ROUTER_URL']; -// if the environment is a 'test_' environment, then we'll use an -// ephemeral database -/*if (config.get('env').substr(0,5) === 'test_') {*/ - //if (config.get('database').driver === 'mysql') { - //process.env['DATABASE_NAME'] = - //process.env['DATABASE_NAME'] || "browserid_tmp_" + secrets.generate(6); - //console.log("temp mysql database:", process.env['DATABASE_NAME']); - //} else if (config.get('database').driver === 'json') { - //process.env['DATABASE_NAME'] = process.env['DATABASE_NAME'] || temp.path({suffix: '.db'}); - //console.log("temp json database:", process.env['DATABASE_NAME']); - //} -/*}*/ - // Windows can't use signals, so lets figure out if we should use them // To force signals, set the environment variable SUPPORTS_SIGNALS=true. // Otherwise, they will be feature-detected.