WIP build server

.awsbox.json

@@ -2,7 +2,8 @@
   "processes": [
     "bin/router",
     "bin/api",
-    "bin/static"
+    "bin/static",
+    "bin/builder"
   ],
   "hooks": {
     "postcreate": "scripts/post_create.sh"

bin/builder

@@ -0,0 +1,277 @@
+#!/usr/bin/env node
+
+const temp    = require('temp');
+const path    = require('path');
+const util    = require('util');
+const events  = require('events');
+const http    = require('http');
+const git     = require('awsbox/lib/git.js');
+const fs      = require('fs');
+const express = require('express');
+const irc     = require('irc');
+const config  = require('../etc/config');
+const spawn   = require('child_process').spawn;
+
+console.log("build server starting up");
+
+var buildScript = path.resolve(__dirname, '..', 'script', 'crxmake.sh');
+var downloadDir = path.join(__dirname, '..', 'downloads');
+
+var githubRepo  = 'git://github.com/mozilla/skycrane';
+var devBranch   = 'master';
+var downloadFile = 'latest.crx';
+
+var ircChannel = '#tobmog';
+
+// a class capable of deploying and emmitting events along the way
+function Builder(options) {
+  events.EventEmitter.call(this);
+  if (!options) optiosn = {};
+
+  this.repo = options.repo;
+  this.branch = options.branch;
+
+  // a directory where we'll keep code
+  this._codeDir = process.env['CODE_DIR'] || temp.mkdirSync();
+  console.log("code dir is:", this._codeDir);
+  var self = this;
+
+  git.init(this._codeDir, function(err) {
+    if (err) {
+      console.log("can't init code dir:", err);
+      process.exit(1);
+    }
+    self.emit('ready');
+  });
+}
+
+util.inherits(Builder, events.EventEmitter);
+
+Builder.prototype._writeSha = function(sha, cb) {
+  var self = this;
+  fs.writeFile(path.join(downloadDir, 'ver.txt'), sha, function(err, sha) {
+    if (err) self.emit('info', 'could not write last sha');
+    if (cb) cb(err);
+  });
+};
+
+Builder.prototype._getLatestRunningSHA = function(cb) {
+  var self = this;
+  fs.readFile(path.join(downloadDir, 'ver.txt'), 'utf8', function(err, sha) {
+    if (err) self.emit('info', 'could not get last sha');
+    if (cb) cb(err, sha);
+  });
+};
+
+Builder.prototype._buildNewCode = function(cb) {
+  var self = this;
+
+  function splitAndEmit(chunk) {
+    if (chunk) chunk = chunk.toString();
+    if (typeof chunk === 'string') {
+      chunk.split('\n').forEach(function (line) {
+        line = line.trim();
+        if (line.length) self.emit('progress', line);
+      });
+    }
+  }
+
+  console.log(buildScript);
+  var crxBuild = spawn(buildScript, [ self._codeDir, '~/.ssh/id_rsa' ], { cwd: self._codeDir });
+
+  crxBuild.stdout.on('data', splitAndEmit);
+  crxBuild.stderr.on('data', splitAndEmit);
+
+  crxBuild.on('exit', function(code, signal) {
+    if (code != 0) {
+      cb('could not build crx');
+      return;
+    }
+  });
+};
+
+Builder.prototype._pullLatest = function(cb) {
+  var self = this;
+  git.pull(this._codeDir, this.repo, this.branch, function(l) {
+    self.emit('progress', l);
+  }, function(err) {
+    if (err) return cb(err);
+    git.currentSHA(self._codeDir, function(err, latest) {
+      if (err) return cb(err);
+      self.emit('info', 'latest available sha is ' + latest);
+      self._getLatestRunningSHA(function(err, running) {
+        if (latest !== running) {
+          self.emit('deployment_begins', {
+            sha: latest,
+          });
+          var startTime = new Date();
+
+          self._buildNewCode(function(err, res) {
+            if (err) return cb(err);
+            self._writeSha(latest, function(err) {
+              // deployment is complete!
+              self.emit('deployment_complete', {
+                sha: latest,
+                time: (new Date() - startTime)
+              });
+              cb(null, null);
+            });
+          });
+        } else {
+          self.emit('info', 'up to date');
+          cb(null, null);
+        }
+      });
+    });
+  });
+}
+
+// may be invoked any time we suspect updates have occured to re-deploy
+// if needed
+Builder.prototype.checkForUpdates = function() {
+  var self = this;
+
+  if (this._busy) {
+    self.emit('info', 'busy');
+    return;
+  }
+
+  this._busy = true;
+  self.emit('info', 'checking for updates');
+
+  self._pullLatest(function(err, sha) {
+    if (err) self.emit('error', err);
+    self.emit('info', 'done checking');
+    self._busy = false;
+  });
+}
+
+// create dev builder
+var builder = new Builder({ repo: githubRepo, branch: devBranch });
+
+var currentLogFile = null;
+// a directory where we'll keep deployment logs
+var deployLogDir = process.env['DEPLOY_LOG_DIR'] || temp.mkdirSync();
+
+var deployingSHA = null;
+
+console.log("deployment log dir is:", deployLogDir);
+
+[ 'info', 'ready', 'error', 'deployment_begins', 'deployment_complete', 'progress' ].forEach(function(evName) {
+  builder.on(evName, function(data) {
+    if (data !== null && data !== undefined && typeof data != 'string') data = JSON.stringify(data, null, 2);
+    var msg = evName + (data ? (": " + data) : "")
+    console.log(msg)
+    if (currentLogFile) currentLogFile.write(msg + "\n");
+  });
+});
+
+// irc integration!
+var ircClient = null;
+function ircSend(msg) {
+  if (!ircClient) {
+    ircClient = new irc.Client('irc.mozilla.org', 'gombot_builder', {
+      channels: [ircChannel]
+    });
+    ircClient.on('error', function(e) {
+      console.log('irc error: ', e);
+    });
+    ircClient.once('join' + ircChannel, function(e) {
+      ircClient.say(ircChannel, msg);
+    });
+  } else {
+    ircClient.say(ircChannel, msg);
+  }
+}
+
+function ircDisconnect() {
+  setTimeout(function() {
+    if (ircClient) {
+      ircClient.disconnect();
+      ircClient = null;
+    }
+  }, 1000);
+}
+
+
+// now when deployment begins, we log all events
+builder.on('deployment_begins', function(r) {
+  currentLogFile = fs.createWriteStream(path.join(deployLogDir, r.sha + ".txt"));
+  currentLogFile.write("deployment of " + r.sha + " begins\n");
+  deployingSHA = r.sha;
+  ircSend("deploying " + r.sha);
+});
+
+function closeLogFile() {
+  if (currentLogFile) {
+    currentLogFile.end();
+    currentLogFile = null;
+  }
+}
+
+builder.on('deployment_complete', function(r) {
+  ircSend("deployment of " + deployingSHA + " completed successfully in " +
+          (r.time / 1000.0).toFixed(2) + "s");
+  ircDisconnect();
+
+  closeLogFile();
+  deployingSHA = null;
+
+  // always check to see if we should try another deployment after one succeeds to handle
+  // rapid fire commits
+  console.log('from complete');
+  builder.checkForUpdates();
+});
+
+builder.on('error', function(r) {
+  ircSend("deployment of " + deployingSHA + " failed.  check logs for deets");
+  ircDisconnect();
+
+  closeLogFile();
+  deployingSHA = null;
+
+  // on error, try again in 2 minutes
+  setTimeout(function () {
+    console.log('from error');
+    builder.checkForUpdates();
+  }, 2 * 60 * 1000);
+});
+
+
+// We check every 15 minutes, in case a cosmic ray hits and github's
+// webhooks fail, or other unexpected errors occur
+setInterval(function () {
+  console.log('from interval');
+  builder.checkForUpdates();
+}, (1000 * 60 * 15));
+
+// check for updates at startup
+builder.on('ready', function() {
+  console.log('from ready');
+  builder.checkForUpdates();
+});
+
+// setup build server
+var app = express();
+var server = http.createServer(app);
+
+var check = function(req, res) {
+  console.log('from check');
+  builder.checkForUpdates();
+  res.send('ok');
+};
+
+app.get('/check', check);
+app.post('/check', check);
+
+app.get('/', function(req, res) {
+  var what = "idle";
+  if (deployingSHA) what = "deploying " + deployingSHA;
+  res.send(what);
+});
+
+app.use(express.static(deployLogDir));
+
+server.listen(config.process.builder.port, config.process.builder.host, function() {
+  console.log("running on http://" + server.address().address + ":" + server.address().port);
+});

bin/router

@@ -12,8 +12,9 @@ const forward = require('../lib/http_forward').forward;
 var app = express();
 var server = http.createServer(app);
 
-var api_url    = config.api_url;
+var api_url     = config.api_url;
-var static_url = config.static_url;
+var static_url  = config.static_url;
+var builder_url = config.builder_url + '/check';
 
 console.log("router starting up");
 
@@ -23,8 +24,28 @@ app.use(express.logger());
 // redirect requests to the "verifier" processes
 app.use(function(req, res, next) {
   if (/^\/api/.test(req.url)) {
-    forward(
+    forward(api_url + req.url.replace(/^\/api/,''),
-      api_url+req.url.replace(/^\/api/,''), req, res,
+      req, res,
+      function(err) {
+        if (err) {
+          console.error("error forwarding request:", err);
+          req.destroy();
+        }
+      });
+  } else {
+    return next();
+  }
+});
+
+// kick-off addon updates
+app.use(function(req, res, next) {
+  if (req.url === '/update_addons') {
+    // grab addon code from github
+    // run extension packager script
+    // issue update to installed extensions
+    //  - dev: updated every push
+    //  - nightly: updated every day to last dev
+    forward(builder_url, req, res,
       function(err) {
         if (err) {
           console.error("error forwarding request:", err);
@@ -38,8 +59,7 @@ app.use(function(req, res, next) {
 
 //static catch-all
 app.use(function(req, res, next) {
-  forward(
+  forward(static_url + req.url, req, res,
-    static_url+req.url, req, res,
     function(err) {
       if (err) {
         console.error("error forwarding request:", err);

etc/config.js

@@ -13,6 +13,10 @@ var config = module.exports = {
     static: {
       port: 20002,
       host: '127.0.0.1'
+    },
+    builder: {
+      port: 20003,
+      host: '127.0.0.1'
     }
   },
   hapi: {
@@ -29,3 +33,4 @@ var config = module.exports = {
 
 config.api_url = 'http://' + config.process.api.host + ':' + config.process.api.port;
 config.static_url = 'http://' + config.process.static.host + ':' + config.process.static.port;
+config.builder_url = 'http://' + config.process.builder.host + ':' + config.process.builder.port;

package.json

@@ -7,7 +7,8 @@
     "hapi": "git://github.com/lloyd/hapi#7e6cf0",
     "walkdir": "0.0.5",
     "express": "3.0.2",
-    "nunjucks": "0.1.5"
+    "nunjucks": "0.1.5",
+    "irc": "0.3.3"
   },
   "optionalDependencies": {
     "couchbase": "0.0.4"

scripts/crxmake.sh

@@ -0,0 +1,44 @@
+#!/bin/bash -e
+#
+# Purpose: Pack a Chromium extension directory into crx format
+
+if test $# -ne 2; then
+  echo "Usage: crxmake.sh <extension dir> <pem path>"
+  exit 1
+fi
+
+dir=$1
+key=$2 || ~/.ssh/id_rsa
+name=$(basename "$dir")
+crx="$name.crx"
+pub="$name.pub"
+sig="$name.sig"
+zip="$name.zip"
+trap 'rm -f "$pub" "$sig" "$zip"' EXIT
+
+echo "oh hai"
+
+# zip up the crx dir
+cwd=$(pwd -P)
+(cd "$dir" && zip -qr -9 -X "$cwd/$zip" .)
+
+# signature
+openssl sha1 -sha1 -binary -sign "$key" < "$zip" > "$sig"
+
+# public key
+openssl rsa -pubout -outform DER < "$key" > "$pub" 2>/dev/null
+
+byte_swap () {
+  # Take "abcdefgh" and return it as "ghefcdab"
+  echo "${1:6:2}${1:4:2}${1:2:2}${1:0:2}"
+}
+
+crmagic_hex="4372 3234" # Cr24
+version_hex="0200 0000" # 2
+pub_len_hex=$(byte_swap $(printf '%08x\n' $(ls -l "$pub" | awk '{print $5}')))
+sig_len_hex=$(byte_swap $(printf '%08x\n' $(ls -l "$sig" | awk '{print $5}')))
+(
+  echo "$crmagic_hex $version_hex $pub_len_hex $sig_len_hex" | xxd -r -p
+  cat "$pub" "$sig" "$zip"
+) > "$crx"
+echo "Wrote $crx"

scripts/run_locally.js

@@ -13,9 +13,13 @@ const HOST = process.env['GOMBOT_IP_ADDRESS'] || process.env['GOMBOT_HOST'] || "
 var daemonsToRun = {
   api: { },
   static: { },
+  builder: { },
   router: { }
 };
 
+// only run builder if specified
+//if (!process.env.BUILD_SERVER) delete daemonsToRun.builder;
+
 process.env['GOMBOT_HOST'] = HOST;
 
 // use the "local" configuration
@@ -32,22 +36,10 @@ process.env['LOG_TO_CONSOLE'] = 1;
 process.env['GOMBOT_ROUTER_URL'] = 'http://' + HOST + ":20000";
 process.env['GOMBOT_API_URL']    = 'http://' + HOST + ":20001";
 process.env['GOMBOT_STATIC_URL'] = 'http://' + HOST + ":20002";
+process.env['GOMBOT_BUILDER_URL'] = 'http://' + HOST + ":20003";
 
 process.env['PUBLIC_URL'] = process.env['GOMBOT_ROUTER_URL'];
 
-// if the environment is a 'test_' environment, then we'll use an
-// ephemeral database
-/*if (config.get('env').substr(0,5) === 'test_') {*/
-  //if (config.get('database').driver === 'mysql') {
-    //process.env['DATABASE_NAME'] =
-      //process.env['DATABASE_NAME'] || "browserid_tmp_" + secrets.generate(6);
-    //console.log("temp mysql database:", process.env['DATABASE_NAME']);
-  //} else if (config.get('database').driver === 'json') {
-    //process.env['DATABASE_NAME'] =  process.env['DATABASE_NAME'] || temp.path({suffix: '.db'});
-    //console.log("temp json database:", process.env['DATABASE_NAME']);
-  //}
-/*}*/
-
 // Windows can't use signals, so lets figure out if we should use them
 // To force signals, set the environment variable SUPPORTS_SIGNALS=true.
 // Otherwise, they will be feature-detected.