Merge pull request mozilla/fxa-auth-server#26 from dannycoates/heka

heka experiment

packages/fxa-auth-server/.awsbox.json

@@ -1,12 +1,17 @@
 {
-  "ami": "ami-6f690106",
+  "ami": "ami-91a2c8f8",
   "processes": [
     "index.js"
   ],
   "env": {
-    "CONFIG_FILES": "$HOME/code/config/aws.json,$HOME/config.json"
+    "CONFIG_FILES": "$HOME/code/config/aws.json,$HOME/config.json",
+    "HEKAD_CONFIG": "$HOME/code/heka/hekad.toml"
   },
   "remote_hooks": {
-    "postcreate": "scripts/aws/post_create.sh"
-  }
-}
+    "postcreate": "scripts/aws/post_create.sh",
+    "postdeploy": "scripts/aws/post_deploy.sh"
+  },
+  "packages": [
+    "mysql-server"
+  ]
+}

packages/fxa-auth-server/.gitignore

@@ -1,4 +1,6 @@
 /node_modules
+/sandbox
+/dashboard
 /config/public-key.json
 /config/secret-key.json
 *.swp

packages/fxa-auth-server/heka/hekad.toml

@@ -0,0 +1,38 @@
+[UdpInput]
+address = "127.0.0.1:4880"
+
+[StatsdInput]
+address = ":8125"
+
+[TCP:5565]
+type = "TcpInput"
+address = ":5565"
+    [TCP:5565.signer.test_0]
+    hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm"
+
+[TestSandboxManager]
+type = "SandboxManagerFilter"
+message_signer = "test"
+message_matcher = "Type == 'heka.control.sandbox'"
+working_directory = "./sandbox"
+max_filters = 100
+
+[memory_counter]
+type = "SandboxFilter"
+message_matcher = "Type == 'statmetric'"
+ticker_interval = 15
+script_type = "lua"
+filename = "./heka/memory.lua"
+preserve_data = false
+memory_limit = 1048576
+instruction_limit = 100
+output_limit = 64512
+
+[debug_output]
+type = "LogOutput"
+message_matcher = "Type != 'heka.all-report' && Type != 'heka.sandbox-output'"
+payload_only = false
+
+[DashboardOutput]
+ticker_interval = 15
+message_matcher = "Type == 'heka.all-report' || Type == 'heka.sandbox-terminated' || (Type == 'heka.sandbox-output' && Fields[payload_type] == 'cbuf')"

packages/fxa-auth-server/heka/memory.lua

@@ -0,0 +1,25 @@
+data = circular_buffer.new(240, 3, 15)
+local cols = {
+  rss = data:set_header(1, "RSS", "max"),
+  heapTotal = data:set_header(2, "HEAP_TOTAL", "max"),
+  heapUsed = data:set_header(3, "HEAP_USED", "max")
+}
+
+function process_message()
+  local ts = read_message("Timestamp")
+  local payload = read_message("Payload")
+  if payload == nil then return 0 end
+
+  for k, v in string.gmatch(payload, "stats.(%w+) (%d+)") do
+    if cols[k] ~= nil then
+      data:set(ts, cols[k], v)
+    end
+  end
+
+  return 0
+end
+
+function timer_event(ns)
+  output(data)
+  inject_message("cbuf", "Process Memory Usage")
+end

packages/fxa-auth-server/heka/sbmgr.toml

@@ -0,0 +1,6 @@
+ip_address          = "127.0.0.1:5565"
+[signer]
+    name         = "test"
+    hmac_hash    = "md5"
+    hmac_key     = "4865ey9urgkidls xtb0[7lf9rzcivthkm"
+    version      = 0

packages/fxa-auth-server/lib/kv.js

@@ -2,6 +2,6 @@ var config = require('./config');
 var kvstore = require('kvstore')(config.root());
 
 module.exports = {
-	cache: kvstore.connect({ backend: config.get('kvstore.cache') }),
-	store: kvstore.connect({ backend: config.get('kvstore.backend' )})
+  cache: kvstore.connect({ backend: config.get('kvstore.cache') }),
+  store: kvstore.connect({ backend: config.get('kvstore.backend' )})
 };

packages/fxa-auth-server/lib/stats.js

@@ -0,0 +1,56 @@
+const os = require('os');
+const Heka = require('heka');
+const Statsd = require('node-statsd').StatsD;
+
+//TODO read config
+
+const HOSTNAME = os.hostname();
+const PID = process.pid;
+
+// var heka = Heka.clientFromJsonConfig(
+//   JSON.stringify(
+//     {
+//       sender: {
+//         factory: './senders:udpSenderFactory',
+//         hosts: '127.0.0.1',
+//         ports: 4880
+//       },
+//       logger: 'picl-idp',
+//       severity: 5
+//     }
+//   )
+// );
+
+var statsd = new Statsd(
+  {
+    host: '127.0.0.1',
+    port: 8125
+  }
+);
+
+module.exports = {
+  ops: function (event) {
+
+    statsd.gauge('rss', event.proc.mem.rss);
+    statsd.gauge('heapTotal', event.proc.mem.heapTotal);
+    statsd.gauge('heapUsed', event.proc.mem.heapUsed);
+
+    // When we send stats with heka it will look like this:
+    //
+    // heka.heka(
+    //  'ops',
+    //  {
+    //    timestamp: new Date(event.timestamp),
+    //    severity: 6,
+    //    fields: {
+    //      rss: event.proc.mem.rss,
+    //      heapTotal: event.proc.mem.heapTotal,
+    //      heapUsed: event.proc.mem.heapUsed,
+    //      uptime: event.proc.uptime
+    //    },
+    //    pid: PID,
+    //    hostname: HOSTNAME
+    //  }
+    // );
+  }
+};

packages/fxa-auth-server/package.json

@@ -8,7 +8,7 @@
   },
   "scripts": {
     "test": "CONFIG_FILES=./test/config/test.json ./node_modules/.bin/mocha -R spec --recursive --timeout 10000",
-    "start": "NODE_ENV=local node index.js"
+    "start": "scripts/start-local.sh"
   },
   "repository": "",
   "license": "MPL 2.0",
@@ -26,10 +26,12 @@
     "uuid": "1.4.1",
     "async": "0.2.8",
     "kvstore": "git://github.com/mozilla/node-kvstore.git#4b8c2f6763",
-    "good": "0.5.7"
+    "good": "git://github.com/dannycoates/good.git#223e5d6db3",
+    "heka": "git://github.com/dannycoates/heka-node.git#dc31a1c1f4",
+    "node-statsd": "0.0.7"
   },
   "devDependencies": {
-    "awsbox": "git://github.com/dannycoates/awsbox.git",
+    "awsbox": "0.4.5",
     "mocha": "1.9.x",
     "request": "2.21.x",
     "jshint": "0.9.1"

packages/fxa-auth-server/routes/heartbeat.js

@@ -16,17 +16,17 @@ exports.routes = [
 ];
 
 function heartbeat(request) {
-	async.each(
-		[kv.store, kv.cache],
-		function (db, done) {
-			db.ping(done);
-		},
-		function (err) {
-			var text = 'ok';
-			if (err) {
-				text = err.toString();
-			}
-			request.reply(text).type('text/plain');
-		}
-	);
+  async.each(
+    [kv.store, kv.cache],
+    function (db, done) {
+      db.ping(done);
+    },
+    function (err) {
+      var text = 'ok';
+      if (err) {
+        text = err.toString();
+      }
+      request.reply(text).type('text/plain');
+    }
+  );
 }

packages/fxa-auth-server/scripts/aws/post_create.sh

@@ -6,8 +6,3 @@ sudo /sbin/service mysqld start
 echo "CREATE USER 'picl'@'localhost';" | mysql -u root
 echo "CREATE DATABASE picl;" | mysql -u root
 echo "GRANT ALL ON picl.* TO 'picl'@'localhost';" | mysql -u root
-
-echo "hacking iptables"
-# this is a workaround until someone figures
-# out why the iptables on this image don't stick
-sudo /home/app/scripts/setup_routes.sh

packages/fxa-auth-server/scripts/aws/post_deploy.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+echo "Restarting heka"
+
+pid=`ps -aefw | grep "hekad" | grep -v " grep " | awk '{print $2}'`
+if [[ $pid ]] ; then
+  kill -s INT $pid
+fi
+cd /home/app/code
+nohup hekad -config=$HEKAD_CONFIG > /dev/null 2>&1 &
+
+echo "DONE"

packages/fxa-auth-server/scripts/hekad.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+#
+# chkconfig: 35 99 99
+# description: Heka Daemon
+#
+
+. /etc/rc.d/init.d/functions
+
+USER="app"
+
+DAEMON="/usr/local/bin/hekad"
+ROOT_DIR="/home/app"
+
+CONFIG_FILE=$HEKAD_CONFIG
+: ${CONFIG_FILE:="/etc/hekad.toml"}
+
+LOG_FILE="$ROOT_DIR/hekad.log"
+
+LOCK_FILE="/var/lock/subsys/hekad"
+
+do_start()
+{
+  if [ ! -f "$LOCK_FILE" ] ; then
+    echo -n $"Starting hekad: "
+    runuser -l "$USER" -c "$DAEMON -config=$CONFIG_FILE >> $LOG_FILE &" && echo_success || echo_failure
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && touch $LOCK_FILE
+  else
+    echo "hekad is locked."
+    RETVAL=1
+  fi
+}
+do_stop()
+{
+  echo -n $"Stopping hekad: "
+  pid=`ps -aefw | grep "$DAEMON" | grep -v " grep " | awk '{print $2}'`
+  runuser -l "$USER" -c "kill -SIGINT $pid > /dev/null 2>&1" && echo_success || echo_failure
+  RETVAL=$?
+  echo
+  [ $RETVAL -eq 0 ] && rm -f $LOCK_FILE
+}
+
+case "$1" in
+  start)
+    do_start
+    ;;
+  stop)
+    do_stop
+    ;;
+  restart)
+    do_stop
+    do_start
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|restart}"
+    RETVAL=1
+esac
+
+exit $RETVAL

packages/fxa-auth-server/scripts/start-local.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+NODE_ENV="local"
+
+pid=`ps -aefw | grep "hekad" | grep -v " grep " | awk '{print $2}'`
+if [[ $pid ]] ; then
+  echo "stopping heka"
+  kill -s INT $pid
+fi
+echo "starting heka"
+hekad -config=heka/hekad.toml &
+
+node index.js

packages/fxa-auth-server/server.js

@@ -6,6 +6,7 @@ const Hapi = require('hapi');
 
 const config = require('./lib/config');
 const routes = require('./routes');
+const stats = require('./lib/stats');
 
 // server settings
 var settings = {
@@ -49,14 +50,19 @@ server.ext('onPreResponse', function (request, next) {
 
 server.pack.require('good', {
     subscribers: {
-      console: ['ops', 'request', 'log']
+      console: ['request', 'log'],
+      ops: {
+        events: ['ops'],
+        handler: stats.ops
+      }
     },
     extendedRequests: true,
     leakDetection: true
   },
   function(err) {
     if (err) server.log(['error'], err);
-  });
+  }
+);
 
 //TODO throttle extension
 //TODO toobusy extension