From a7276cd0ac6234b63f97ac1570704809a58d3e0a Mon Sep 17 00:00:00 2001 From: Lloyd Hilaiel Date: Fri, 30 Mar 2012 14:07:48 -0600 Subject: [PATCH] import of some tasty nodejs libraries --- lib/aws.js | 7 ++++ lib/dns.js | 83 ++++++++++++++++++++++++++++++++++++ lib/git.js | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/key.js | 57 +++++++++++++++++++++++++ lib/sec.js | 59 ++++++++++++++++++++++++++ lib/ssh.js | 43 +++++++++++++++++++ lib/vm.js | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 490 insertions(+) create mode 100644 lib/aws.js create mode 100644 lib/dns.js create mode 100644 lib/git.js create mode 100644 lib/key.js create mode 100644 lib/sec.js create mode 100644 lib/ssh.js create mode 100644 lib/vm.js diff --git a/lib/aws.js b/lib/aws.js new file mode 100644 index 0000000..6641989 --- /dev/null +++ b/lib/aws.js @@ -0,0 +1,7 @@ +const +awslib = require('aws-lib'); + +module.exports = awslib.createEC2Client(process.env['AWS_ID'], process.env['AWS_SECRET'], { + version: '2011-12-15' +}); + diff --git a/lib/dns.js b/lib/dns.js new file mode 100644 index 0000000..99a2f58 --- /dev/null +++ b/lib/dns.js @@ -0,0 +1,83 @@ +const +http = require('http'), +xml2js = new (require('xml2js')).Parser(), +jsel = require('JSONSelect'); + +const envVar = 'BROWSERID_DEPLOY_DNS_KEY'; +if (!process.env[envVar]) { + throw "Missing api key! contact lloyd and set the key in your env: " + + envVar; +} + +const api_key = process.env[envVar]; + +function doRequest(method, path, body, cb) { + var req = http.request({ + auth: 'lloyd@hilaiel.com:' + api_key, + host: 'ns.zerigo.com', + port: 80, + path: path, + method: method, + headers: { + 'Content-Type': 'application/xml', + 'Content-Length': body ? body.length : 0 + } + }, function(r) { + if ((r.statusCode / 100).toFixed(0) != 2 && + r.statusCode != 404) { + return cb("non 200 status: " + r.statusCode); + } + buf = ""; + r.on('data', function(chunk) { + buf += chunk; + }); + r.on('end', function() { + xml2js.parseString(buf, cb); + }); + }); + if (body) req.write(body); + req.end(); +}; + +exports.updateRecord = function (hostname, zone, ip, cb) { + doRequest('GET', '/api/1.1/zones.xml', null, function(err, r) { + if (err) return cb(err); + var m = jsel.match('object:has(:root > .domain:val(?)) > .id .#', + [ zone ], r); + if (m.length != 1) return cb("couldn't extract domain id from zerigo"); + var path = '/api/1.1/hosts.xml?zone_id=' + m[0]; + var body = '' + ip + 'A'; + body += '' + hostname + '' + body += ''; + doRequest('POST', path, body, function(err, r) { + cb(err); + }); + }); +}; + +exports.deleteRecord = function (hostname, cb) { + doRequest('GET', '/api/1.1/hosts.xml?fqdn=' + hostname, null, function(err, r) { + if (err) return cb(err); + var m = jsel.match('.host .id > .#', r); + if (!m.length) return cb("no such DNS record"); + function deleteOne() { + if (!m.length) return cb(null); + var one = m.shift(); + doRequest('DELETE', '/api/1.1/hosts/' + one + '.xml', null, function(err) { + if (err) return cb(err); + deleteOne(); + }); + } + deleteOne(); + }); +}; + +exports.inUse = function (hostname, cb) { + doRequest('GET', '/api/1.1/hosts.xml?fqdn=' + hostname, null, function(err, r) { + if (err) return cb(err); + var m = jsel.match('.host', r); + // we shouldn't have multiple! oops! let's return the first one + if (m.length) return cb(null, m[0]); + cb(null, null); + }); +} diff --git a/lib/git.js b/lib/git.js new file mode 100644 index 0000000..4fc20d1 --- /dev/null +++ b/lib/git.js @@ -0,0 +1,120 @@ +const +child_process = require('child_process'); +spawn = child_process.spawn, +path = require('path'); + +exports.addRemote = function(name, host, cb) { + var cmd = 'git remote add ' + name + ' app@'+ host + ':git'; + child_process.exec(cmd, cb); +}; + +// remove a remote, but only if it is pointed to a specific +// host. This will keep deploy from killing manuall remotes +// that you've set up +exports.removeRemote = function(name, host, cb) { + var desired = 'app@'+ host + ':git'; + var cmd = 'git remote -v show | grep push'; + child_process.exec(cmd, function(err, r) { + try { + var remotes = {}; + r.split('\n').forEach(function(line) { + if (!line.length) return; + var line = line.split('\t'); + if (!line.length == 2) return; + remotes[line[0]] = line[1].split(" ")[0]; + }); + if (remotes[name] && remotes[name] === desired) { + child_process.exec('git remote rm ' + name, cb); + } else { + throw "no such remote"; + } + } catch(e) { + cb(e); + } + }); +}; + +exports.currentSHA = function(dir, cb) { + if (typeof dir === 'function' && cb === undefined) { + cb = dir; + dir = path.join(__dirname, '..', '..'); + } + console.log(dir); + + var p = spawn('git', [ 'log', '--pretty=%h', '-1' ], { + env: { GIT_DIR: path.join(dir, ".git") } + }); + var buf = ""; + p.stdout.on('data', function(d) { + buf += d; + }); + p.on('exit', function(code, signal) { + console.log(buf); + var gitsha = buf.toString().trim(); + if (gitsha && gitsha.length === 7) { + return cb(null, gitsha); + } + cb("can't extract git sha from " + dir); + }); +}; + +function splitAndEmit(chunk, cb) { + if (chunk) chunk = chunk.toString(); + if (typeof chunk === 'string') { + chunk.split('\n').forEach(function (line) { + line = line.trim(); + if (line.length) cb(line); + }); + } +} + +exports.push = function(dir, host, pr, cb) { + if (typeof host === 'function' && cb === undefined) { + cb = pr; + pr = host; + host = dir; + dir = path.join(__dirname, '..', '..'); + } + + var p = spawn('git', [ 'push', 'app@' + host + ":git", 'dev:master' ], { + env: { + GIT_DIR: path.join(dir, ".git"), + GIT_WORK_TREE: dir + } + }); + p.stdout.on('data', function(c) { splitAndEmit(c, pr); }); + p.stderr.on('data', function(c) { splitAndEmit(c, pr); }); + p.on('exit', function(code, signal) { + return cb(code = 0); + }); +}; + +exports.pull = function(dir, remote, branch, pr, cb) { + var p = spawn('git', [ 'pull', "-f", remote, branch + ":" + branch ], { + env: { + GIT_DIR: path.join(dir, ".git"), + GIT_WORK_TREE: dir, + PWD: dir + }, + cwd: dir + }); + + p.stdout.on('data', function(c) { splitAndEmit(c, pr); }); + p.stderr.on('data', function(c) { splitAndEmit(c, pr); }); + + p.on('exit', function(code, signal) { + return cb(code = 0); + }); +} + +exports.init = function(dir, cb) { + var p = spawn('git', [ 'init' ], { + env: { + GIT_DIR: path.join(dir, ".git"), + GIT_WORK_TREE: dir + } + }); + p.on('exit', function(code, signal) { + return cb(code = 0); + }); +}; diff --git a/lib/key.js b/lib/key.js new file mode 100644 index 0000000..d93da01 --- /dev/null +++ b/lib/key.js @@ -0,0 +1,57 @@ +const +aws = require('./aws.js'), +path = require('path'), +fs = require('fs'), +child_process = require('child_process'), +jsel = require('JSONSelect'), +crypto = require('crypto'); + +const keyPath = process.env['PUBKEY'] || path.join(process.env['HOME'], ".ssh", "id_rsa.pub"); + +exports.read = function(cb) { + fs.readFile(keyPath, cb); +}; + +exports.fingerprint = function(cb) { + exports.read(function(err, buf) { + if (err) return cb(err); + var b = new Buffer(buf.toString().split(' ')[1], 'base64'); + var md5sum = crypto.createHash('md5'); + md5sum.update(b); + cb(null, md5sum.digest('hex')); + }); +/* + child_process.exec( + "ssh-keygen -lf " + keyPath, + function(err, r) { + if (!err) r = r.split(' ')[1]; + cb(err, r); + }); +*/ +}; + +exports.getName = function(cb) { + exports.fingerprint(function(err, fingerprint) { + if (err) return cb(err); + + var keyName = "browserid deploy key (" + fingerprint + ")"; + + // is this fingerprint known? + aws.call('DescribeKeyPairs', {}, function(result) { + var found = jsel.match(":has(.keyName:val(?)) > .keyName", [ keyName ], result); + if (found.length) return cb(null, keyName); + + // key isn't yet installed! + exports.read(function(err, key) { + aws.call('ImportKeyPair', { + KeyName: keyName, + PublicKeyMaterial: new Buffer(key).toString('base64') + }, function(result) { + if (!result) return cb('null result from ec2 on key addition'); + if (result.Errors) return cb(result.Errors.Error.Message); + cb(null, keyName); + }); + }); + }); + }); +}; diff --git a/lib/sec.js b/lib/sec.js new file mode 100644 index 0000000..d821169 --- /dev/null +++ b/lib/sec.js @@ -0,0 +1,59 @@ +const +aws = require('./aws.js'); +jsel = require('JSONSelect'), +key = require('./key.js'); + +// every time you change the security group, change this version number +// so new deployments will create a new group with the changes +const SECURITY_GROUP_VERSION = 1; + +function createError(msg, r) { + var m = jsel.match('.Message', r); + if (m.length) msg += ": " + m[0]; + return msg; +} + +exports.getName = function(cb) { + var groupName = "browserid group v" + SECURITY_GROUP_VERSION; + + // is this fingerprint known? + aws.call('DescribeSecurityGroups', { + GroupName: groupName + }, function(r) { + if (jsel.match('.Code:val("InvalidGroup.NotFound")', r).length) { + aws.call('CreateSecurityGroup', { + GroupName: groupName, + GroupDescription: 'A security group for browserid deployments' + }, function(r) { + if (!r || !r.return === 'true') { + return cb(createError('failed to create security group', r)); + } + aws.call('AuthorizeSecurityGroupIngress', { + GroupName: groupName, + "IpPermissions.1.IpProtocol": 'tcp', + "IpPermissions.1.FromPort": 80, + "IpPermissions.1.ToPort": 80, + "IpPermissions.1.IpRanges.1.CidrIp": "0.0.0.0/0", + "IpPermissions.2.IpProtocol": 'tcp', + "IpPermissions.2.FromPort": 22, + "IpPermissions.2.ToPort": 22, + "IpPermissions.2.IpRanges.1.CidrIp": "0.0.0.0/0", + "IpPermissions.3.IpProtocol": 'tcp', + "IpPermissions.3.FromPort": 443, + "IpPermissions.3.ToPort": 443, + "IpPermissions.3.IpRanges.1.CidrIp" : "0.0.0.0/0" + }, function(r) { + if (!r || !r.return === 'true') { + return cb(createError('failed to create security group', r)); + } + cb(null, groupName); + }); + }); + } else { + // already exists? + var m = jsel.match('.securityGroupInfo > .item > .groupName', r); + if (m.length && m[0] === groupName) return cb(null, groupName); + cb(createError('error creating group', r)); + } + }); +}; diff --git a/lib/ssh.js b/lib/ssh.js new file mode 100644 index 0000000..290abf1 --- /dev/null +++ b/lib/ssh.js @@ -0,0 +1,43 @@ +const +child_process = require('child_process'), +temp = require('temp'), +fs = require('fs'); + +const MAX_TRIES = 20; + +exports.copyUpConfig = function(host, config, cb) { + var tries = 0; + temp.open({}, function(err, r) { + fs.writeFileSync(r.path, JSON.stringify(config, null, 4)); + var cmd = 'scp -o "StrictHostKeyChecking no" ' + r.path + ' app@' + host + ":config.json"; + function oneTry() { + child_process.exec(cmd, function(err, r) { + if (err) { + if (++tries > MAX_TRIES) return cb("can't connect via SSH. stupid amazon"); + console.log(" ... nope. not yet. retrying."); + setTimeout(oneTry, 5000); + } else { + cb(); + } + }); + } + oneTry(); + }); +}; + +exports.copySSL = function(host, pub, priv, cb) { + var cmd = 'scp -o "StrictHostKeyChecking no" ' + pub + ' ec2-user@' + host + ":/etc/ssl/certs/hacksign.in.crt"; + child_process.exec(cmd, function(err, r) { + if (err) return cb(err); + var cmd = 'scp -o "StrictHostKeyChecking no" ' + priv + ' ec2-user@' + host + ":/etc/ssl/certs/hacksign.in.key"; + child_process.exec(cmd, function(err, r) { + var cmd = 'ssh -o "StrictHostKeyChecking no" ec2-user@' + host + " 'sudo /etc/init.d/nginx restart'"; + child_process.exec(cmd, cb); + }); + }); +}; + +exports.addSSHPubKey = function(host, pubkey, cb) { + var cmd = 'ssh -o "StrictHostKeyChecking no" ec2-user@' + host + " 'echo \'" + pubkey + "\' >> .ssh/authorized_keys'"; + child_process.exec(cmd, cb); +}; diff --git a/lib/vm.js b/lib/vm.js new file mode 100644 index 0000000..c08d694 --- /dev/null +++ b/lib/vm.js @@ -0,0 +1,121 @@ +const +aws = require('./aws.js'); +jsel = require('JSONSelect'), +key = require('./key.js'), +sec = require('./sec.js'); + +const BROWSERID_TEMPLATE_IMAGE_ID = 'ami-7e954817'; + +function extractInstanceDeets(horribleBlob) { + var instance = {}; + ["instanceId", "imageId", "instanceState", "dnsName", "keyName", "instanceType", + "ipAddress"].forEach(function(key) { + if (horribleBlob[key]) instance[key] = horribleBlob[key]; + }); + var name = jsel.match('.tagSet :has(.key:val("Name")) > .value', horribleBlob); + if (name.length) { + instance.fullName = name[0]; + // if this is a 'browserid deployment', we'll only display the hostname chosen by the + // user + var m = /^browserid deployment \((.*)\)$/.exec(instance.fullName); + instance.name = m ? m[1] : instance.fullName; + } else { + instance.name = instance.instanceId; + } + return instance; +} + +exports.list = function(cb) { + aws.call('DescribeInstances', {}, function(result) { + var instances = {}; + var i = 1; + jsel.forEach( + '.instancesSet > .item:has(.instanceState .name:val("running"))', + result, function(item) { + var deets = extractInstanceDeets(item); + instances[deets.name || 'unknown ' + i++] = deets; + }); + cb(null, instances); + }); +}; + +exports.destroy = function(name, cb) { + exports.list(function(err, r) { + if (err) return cb('failed to list vms: ' + err); + if (!r[name]) return cb('no such vm'); + + aws.call('TerminateInstances', { + InstanceId: r[name].instanceId + }, function(result) { + try { return cb(result.Errors.Error.Message); } catch(e) {}; + cb(null, r[name]); + }); + }); +}; + +function returnSingleImageInfo(result, cb) { + if (!result) return cb('no results from ec2 api'); + try { return cb(result.Errors.Error.Message); } catch(e) {}; + try { + result = jsel.match('.instancesSet > .item', result)[0]; + cb(null, extractInstanceDeets(result)); + } catch(e) { + return cb("couldn't extract new instance details from ec2 response: " + e); + } +} + +exports.startImage = function(cb) { + key.getName(function(err, keyName) { + if (err) return cb(err); + sec.getName(function(err, groupName) { + if (err) return cb(err); + aws.call('RunInstances', { + ImageId: BROWSERID_TEMPLATE_IMAGE_ID, + KeyName: keyName, + SecurityGroup: groupName, + InstanceType: 't1.micro', + MinCount: 1, + MaxCount: 1 + }, function (result) { + returnSingleImageInfo(result, cb); + }); + }); + }); +}; + +exports.waitForInstance = function(id, cb) { + aws.call('DescribeInstanceStatus', { + InstanceId: id + }, function(r) { + if (!r) return cb('no response from ec2'); + // we're waiting and amazon might not have created the image yet! that's + // not an error, just an api timing quirk + var waiting = jsel.match('.Error .Code:val("InvalidInstanceID.NotFound")', r); + if (waiting.length) { + return setTimeout(function(){ exports.waitForInstance(id, cb); }, 1000); + } + + if (!r.instanceStatusSet) return cb('malformed response from ec2' + JSON.stringify(r, null, 2)); + if (Object.keys(r.instanceStatusSet).length) { + var deets = extractInstanceDeets(r.instanceStatusSet.item); + if (deets && deets.instanceState && deets.instanceState.name === 'running') { + return aws.call('DescribeInstances', { InstanceId: id }, function(result) { + returnSingleImageInfo(result, cb); + }); + } + } + setTimeout(function(){ exports.waitForInstance(id, cb); }, 1000); + }); +}; + +exports.setName = function(id, name, cb) { + aws.call('CreateTags', { + "ResourceId.0": id, + "Tag.0.Key": 'Name', + "Tag.0.Value": name + }, function(result) { + if (result && result.return === 'true') return cb(null); + try { return cb(result.Errors.Error.Message); } catch(e) {}; + return cb('unknown error setting instance name'); + }); +};