зеркало из https://github.com/mozilla/gombot.git
WIP build server
This commit is contained in:
Zachary Carter
Родитель
3675d4df5d
Коммит
5ff97d1f75
|
|
@ -2,7 +2,8 @@
|
|||
"processes": [
|
||||
"bin/router",
|
||||
"bin/api",
|
||||
"bin/static"
|
||||
"bin/static",
|
||||
"bin/builder"
|
||||
],
|
||||
"hooks": {
|
||||
"postcreate": "scripts/post_create.sh"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,277 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const temp = require('temp');
|
||||
const path = require('path');
|
||||
const util = require('util');
|
||||
const events = require('events');
|
||||
const http = require('http');
|
||||
const git = require('awsbox/lib/git.js');
|
||||
const fs = require('fs');
|
||||
const express = require('express');
|
||||
const irc = require('irc');
|
||||
const config = require('../etc/config');
|
||||
const spawn = require('child_process').spawn;
|
||||
|
||||
console.log("build server starting up");
|
||||
|
||||
var buildScript = path.resolve(__dirname, '..', 'script', 'crxmake.sh');
|
||||
var downloadDir = path.join(__dirname, '..', 'downloads');
|
||||
|
||||
var githubRepo = 'git://github.com/mozilla/skycrane';
|
||||
var devBranch = 'master';
|
||||
var downloadFile = 'latest.crx';
|
||||
|
||||
var ircChannel = '#tobmog';
|
||||
|
||||
// a class capable of deploying and emmitting events along the way
|
||||
function Builder(options) {
|
||||
events.EventEmitter.call(this);
|
||||
if (!options) optiosn = {};
|
||||
|
||||
this.repo = options.repo;
|
||||
this.branch = options.branch;
|
||||
|
||||
// a directory where we'll keep code
|
||||
this._codeDir = process.env['CODE_DIR'] || temp.mkdirSync();
|
||||
console.log("code dir is:", this._codeDir);
|
||||
var self = this;
|
||||
|
||||
git.init(this._codeDir, function(err) {
|
||||
if (err) {
|
||||
console.log("can't init code dir:", err);
|
||||
process.exit(1);
|
||||
}
|
||||
self.emit('ready');
|
||||
});
|
||||
}
|
||||
|
||||
util.inherits(Builder, events.EventEmitter);
|
||||
|
||||
Builder.prototype._writeSha = function(sha, cb) {
|
||||
var self = this;
|
||||
fs.writeFile(path.join(downloadDir, 'ver.txt'), sha, function(err, sha) {
|
||||
if (err) self.emit('info', 'could not write last sha');
|
||||
if (cb) cb(err);
|
||||
});
|
||||
};
|
||||
|
||||
Builder.prototype._getLatestRunningSHA = function(cb) {
|
||||
var self = this;
|
||||
fs.readFile(path.join(downloadDir, 'ver.txt'), 'utf8', function(err, sha) {
|
||||
if (err) self.emit('info', 'could not get last sha');
|
||||
if (cb) cb(err, sha);
|
||||
});
|
||||
};
|
||||
|
||||
Builder.prototype._buildNewCode = function(cb) {
|
||||
var self = this;
|
||||
|
||||
function splitAndEmit(chunk) {
|
||||
if (chunk) chunk = chunk.toString();
|
||||
if (typeof chunk === 'string') {
|
||||
chunk.split('\n').forEach(function (line) {
|
||||
line = line.trim();
|
||||
if (line.length) self.emit('progress', line);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(buildScript);
|
||||
var crxBuild = spawn(buildScript, [ self._codeDir, '~/.ssh/id_rsa' ], { cwd: self._codeDir });
|
||||
|
||||
crxBuild.stdout.on('data', splitAndEmit);
|
||||
crxBuild.stderr.on('data', splitAndEmit);
|
||||
|
||||
crxBuild.on('exit', function(code, signal) {
|
||||
if (code != 0) {
|
||||
cb('could not build crx');
|
||||
return;
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
Builder.prototype._pullLatest = function(cb) {
|
||||
var self = this;
|
||||
git.pull(this._codeDir, this.repo, this.branch, function(l) {
|
||||
self.emit('progress', l);
|
||||
}, function(err) {
|
||||
if (err) return cb(err);
|
||||
git.currentSHA(self._codeDir, function(err, latest) {
|
||||
if (err) return cb(err);
|
||||
self.emit('info', 'latest available sha is ' + latest);
|
||||
self._getLatestRunningSHA(function(err, running) {
|
||||
if (latest !== running) {
|
||||
self.emit('deployment_begins', {
|
||||
sha: latest,
|
||||
});
|
||||
var startTime = new Date();
|
||||
|
||||
self._buildNewCode(function(err, res) {
|
||||
if (err) return cb(err);
|
||||
self._writeSha(latest, function(err) {
|
||||
// deployment is complete!
|
||||
self.emit('deployment_complete', {
|
||||
sha: latest,
|
||||
time: (new Date() - startTime)
|
||||
});
|
||||
cb(null, null);
|
||||
});
|
||||
});
|
||||
} else {
|
||||
self.emit('info', 'up to date');
|
||||
cb(null, null);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// may be invoked any time we suspect updates have occured to re-deploy
|
||||
// if needed
|
||||
Builder.prototype.checkForUpdates = function() {
|
||||
var self = this;
|
||||
|
||||
if (this._busy) {
|
||||
self.emit('info', 'busy');
|
||||
return;
|
||||
}
|
||||
|
||||
this._busy = true;
|
||||
self.emit('info', 'checking for updates');
|
||||
|
||||
self._pullLatest(function(err, sha) {
|
||||
if (err) self.emit('error', err);
|
||||
self.emit('info', 'done checking');
|
||||
self._busy = false;
|
||||
});
|
||||
}
|
||||
|
||||
// create dev builder
|
||||
var builder = new Builder({ repo: githubRepo, branch: devBranch });
|
||||
|
||||
var currentLogFile = null;
|
||||
// a directory where we'll keep deployment logs
|
||||
var deployLogDir = process.env['DEPLOY_LOG_DIR'] || temp.mkdirSync();
|
||||
|
||||
var deployingSHA = null;
|
||||
|
||||
console.log("deployment log dir is:", deployLogDir);
|
||||
|
||||
[ 'info', 'ready', 'error', 'deployment_begins', 'deployment_complete', 'progress' ].forEach(function(evName) {
|
||||
builder.on(evName, function(data) {
|
||||
if (data !== null && data !== undefined && typeof data != 'string') data = JSON.stringify(data, null, 2);
|
||||
var msg = evName + (data ? (": " + data) : "")
|
||||
console.log(msg)
|
||||
if (currentLogFile) currentLogFile.write(msg + "\n");
|
||||
});
|
||||
});
|
||||
|
||||
// irc integration!
|
||||
var ircClient = null;
|
||||
function ircSend(msg) {
|
||||
if (!ircClient) {
|
||||
ircClient = new irc.Client('irc.mozilla.org', 'gombot_builder', {
|
||||
channels: [ircChannel]
|
||||
});
|
||||
ircClient.on('error', function(e) {
|
||||
console.log('irc error: ', e);
|
||||
});
|
||||
ircClient.once('join' + ircChannel, function(e) {
|
||||
ircClient.say(ircChannel, msg);
|
||||
});
|
||||
} else {
|
||||
ircClient.say(ircChannel, msg);
|
||||
}
|
||||
}
|
||||
|
||||
function ircDisconnect() {
|
||||
setTimeout(function() {
|
||||
if (ircClient) {
|
||||
ircClient.disconnect();
|
||||
ircClient = null;
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
|
||||
// now when deployment begins, we log all events
|
||||
builder.on('deployment_begins', function(r) {
|
||||
currentLogFile = fs.createWriteStream(path.join(deployLogDir, r.sha + ".txt"));
|
||||
currentLogFile.write("deployment of " + r.sha + " begins\n");
|
||||
deployingSHA = r.sha;
|
||||
ircSend("deploying " + r.sha);
|
||||
});
|
||||
|
||||
function closeLogFile() {
|
||||
if (currentLogFile) {
|
||||
currentLogFile.end();
|
||||
currentLogFile = null;
|
||||
}
|
||||
}
|
||||
|
||||
builder.on('deployment_complete', function(r) {
|
||||
ircSend("deployment of " + deployingSHA + " completed successfully in " +
|
||||
(r.time / 1000.0).toFixed(2) + "s");
|
||||
ircDisconnect();
|
||||
|
||||
closeLogFile();
|
||||
deployingSHA = null;
|
||||
|
||||
// always check to see if we should try another deployment after one succeeds to handle
|
||||
// rapid fire commits
|
||||
console.log('from complete');
|
||||
builder.checkForUpdates();
|
||||
});
|
||||
|
||||
builder.on('error', function(r) {
|
||||
ircSend("deployment of " + deployingSHA + " failed. check logs for deets");
|
||||
ircDisconnect();
|
||||
|
||||
closeLogFile();
|
||||
deployingSHA = null;
|
||||
|
||||
// on error, try again in 2 minutes
|
||||
setTimeout(function () {
|
||||
console.log('from error');
|
||||
builder.checkForUpdates();
|
||||
}, 2 * 60 * 1000);
|
||||
});
|
||||
|
||||
|
||||
// We check every 15 minutes, in case a cosmic ray hits and github's
|
||||
// webhooks fail, or other unexpected errors occur
|
||||
setInterval(function () {
|
||||
console.log('from interval');
|
||||
builder.checkForUpdates();
|
||||
}, (1000 * 60 * 15));
|
||||
|
||||
// check for updates at startup
|
||||
builder.on('ready', function() {
|
||||
console.log('from ready');
|
||||
builder.checkForUpdates();
|
||||
});
|
||||
|
||||
// setup build server
|
||||
var app = express();
|
||||
var server = http.createServer(app);
|
||||
|
||||
var check = function(req, res) {
|
||||
console.log('from check');
|
||||
builder.checkForUpdates();
|
||||
res.send('ok');
|
||||
};
|
||||
|
||||
app.get('/check', check);
|
||||
app.post('/check', check);
|
||||
|
||||
app.get('/', function(req, res) {
|
||||
var what = "idle";
|
||||
if (deployingSHA) what = "deploying " + deployingSHA;
|
||||
res.send(what);
|
||||
});
|
||||
|
||||
app.use(express.static(deployLogDir));
|
||||
|
||||
server.listen(config.process.builder.port, config.process.builder.host, function() {
|
||||
console.log("running on http://" + server.address().address + ":" + server.address().port);
|
||||
});
|
||||
32
bin/router
32
bin/router
|
|
@ -12,8 +12,9 @@ const forward = require('../lib/http_forward').forward;
|
|||
var app = express();
|
||||
var server = http.createServer(app);
|
||||
|
||||
var api_url = config.api_url;
|
||||
var static_url = config.static_url;
|
||||
var api_url = config.api_url;
|
||||
var static_url = config.static_url;
|
||||
var builder_url = config.builder_url + '/check';
|
||||
|
||||
console.log("router starting up");
|
||||
|
||||
|
|
@ -23,8 +24,28 @@ app.use(express.logger());
|
|||
// redirect requests to the "verifier" processes
|
||||
app.use(function(req, res, next) {
|
||||
if (/^\/api/.test(req.url)) {
|
||||
forward(
|
||||
api_url+req.url.replace(/^\/api/,''), req, res,
|
||||
forward(api_url + req.url.replace(/^\/api/,''),
|
||||
req, res,
|
||||
function(err) {
|
||||
if (err) {
|
||||
console.error("error forwarding request:", err);
|
||||
req.destroy();
|
||||
}
|
||||
});
|
||||
} else {
|
||||
return next();
|
||||
}
|
||||
});
|
||||
|
||||
// kick-off addon updates
|
||||
app.use(function(req, res, next) {
|
||||
if (req.url === '/update_addons') {
|
||||
// grab addon code from github
|
||||
// run extension packager script
|
||||
// issue update to installed extensions
|
||||
// - dev: updated every push
|
||||
// - nightly: updated every day to last dev
|
||||
forward(builder_url, req, res,
|
||||
function(err) {
|
||||
if (err) {
|
||||
console.error("error forwarding request:", err);
|
||||
|
|
@ -38,8 +59,7 @@ app.use(function(req, res, next) {
|
|||
|
||||
//static catch-all
|
||||
app.use(function(req, res, next) {
|
||||
forward(
|
||||
static_url+req.url, req, res,
|
||||
forward(static_url + req.url, req, res,
|
||||
function(err) {
|
||||
if (err) {
|
||||
console.error("error forwarding request:", err);
|
||||
|
|
|
|||
|
|
@ -13,6 +13,10 @@ var config = module.exports = {
|
|||
static: {
|
||||
port: 20002,
|
||||
host: '127.0.0.1'
|
||||
},
|
||||
builder: {
|
||||
port: 20003,
|
||||
host: '127.0.0.1'
|
||||
}
|
||||
},
|
||||
hapi: {
|
||||
|
|
@ -29,3 +33,4 @@ var config = module.exports = {
|
|||
|
||||
config.api_url = 'http://' + config.process.api.host + ':' + config.process.api.port;
|
||||
config.static_url = 'http://' + config.process.static.host + ':' + config.process.static.port;
|
||||
config.builder_url = 'http://' + config.process.builder.host + ':' + config.process.builder.port;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
"hapi": "git://github.com/lloyd/hapi#7e6cf0",
|
||||
"walkdir": "0.0.5",
|
||||
"express": "3.0.2",
|
||||
"nunjucks": "0.1.5"
|
||||
"nunjucks": "0.1.5",
|
||||
"irc": "0.3.3"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"couchbase": "0.0.4"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash -e
|
||||
#
|
||||
# Purpose: Pack a Chromium extension directory into crx format
|
||||
|
||||
if test $# -ne 2; then
|
||||
echo "Usage: crxmake.sh <extension dir> <pem path>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
dir=$1
|
||||
key=$2 || ~/.ssh/id_rsa
|
||||
name=$(basename "$dir")
|
||||
crx="$name.crx"
|
||||
pub="$name.pub"
|
||||
sig="$name.sig"
|
||||
zip="$name.zip"
|
||||
trap 'rm -f "$pub" "$sig" "$zip"' EXIT
|
||||
|
||||
echo "oh hai"
|
||||
|
||||
# zip up the crx dir
|
||||
cwd=$(pwd -P)
|
||||
(cd "$dir" && zip -qr -9 -X "$cwd/$zip" .)
|
||||
|
||||
# signature
|
||||
openssl sha1 -sha1 -binary -sign "$key" < "$zip" > "$sig"
|
||||
|
||||
# public key
|
||||
openssl rsa -pubout -outform DER < "$key" > "$pub" 2>/dev/null
|
||||
|
||||
byte_swap () {
|
||||
# Take "abcdefgh" and return it as "ghefcdab"
|
||||
echo "${1:6:2}${1:4:2}${1:2:2}${1:0:2}"
|
||||
}
|
||||
|
||||
crmagic_hex="4372 3234" # Cr24
|
||||
version_hex="0200 0000" # 2
|
||||
pub_len_hex=$(byte_swap $(printf '%08x\n' $(ls -l "$pub" | awk '{print $5}')))
|
||||
sig_len_hex=$(byte_swap $(printf '%08x\n' $(ls -l "$sig" | awk '{print $5}')))
|
||||
(
|
||||
echo "$crmagic_hex $version_hex $pub_len_hex $sig_len_hex" | xxd -r -p
|
||||
cat "$pub" "$sig" "$zip"
|
||||
) > "$crx"
|
||||
echo "Wrote $crx"
|
||||
|
|
@ -13,9 +13,13 @@ const HOST = process.env['GOMBOT_IP_ADDRESS'] || process.env['GOMBOT_HOST'] || "
|
|||
var daemonsToRun = {
|
||||
api: { },
|
||||
static: { },
|
||||
builder: { },
|
||||
router: { }
|
||||
};
|
||||
|
||||
// only run builder if specified
|
||||
//if (!process.env.BUILD_SERVER) delete daemonsToRun.builder;
|
||||
|
||||
process.env['GOMBOT_HOST'] = HOST;
|
||||
|
||||
// use the "local" configuration
|
||||
|
|
@ -32,22 +36,10 @@ process.env['LOG_TO_CONSOLE'] = 1;
|
|||
process.env['GOMBOT_ROUTER_URL'] = 'http://' + HOST + ":20000";
|
||||
process.env['GOMBOT_API_URL'] = 'http://' + HOST + ":20001";
|
||||
process.env['GOMBOT_STATIC_URL'] = 'http://' + HOST + ":20002";
|
||||
process.env['GOMBOT_BUILDER_URL'] = 'http://' + HOST + ":20003";
|
||||
|
||||
process.env['PUBLIC_URL'] = process.env['GOMBOT_ROUTER_URL'];
|
||||
|
||||
// if the environment is a 'test_' environment, then we'll use an
|
||||
// ephemeral database
|
||||
/*if (config.get('env').substr(0,5) === 'test_') {*/
|
||||
//if (config.get('database').driver === 'mysql') {
|
||||
//process.env['DATABASE_NAME'] =
|
||||
//process.env['DATABASE_NAME'] || "browserid_tmp_" + secrets.generate(6);
|
||||
//console.log("temp mysql database:", process.env['DATABASE_NAME']);
|
||||
//} else if (config.get('database').driver === 'json') {
|
||||
//process.env['DATABASE_NAME'] = process.env['DATABASE_NAME'] || temp.path({suffix: '.db'});
|
||||
//console.log("temp json database:", process.env['DATABASE_NAME']);
|
||||
//}
|
||||
/*}*/
|
||||
|
||||
// Windows can't use signals, so lets figure out if we should use them
|
||||
// To force signals, set the environment variable SUPPORTS_SIGNALS=true.
|
||||
// Otherwise, they will be feature-detected.
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче