development; issue #374: Refactor 'make hack' making Vagrantfile and VM more useful

Makefile

@@ -38,8 +38,7 @@ $(DOCKER_BIN): $(DOCKER_DIR)
 
 $(DOCKER_DIR):
 	@mkdir -p $(dir $@)
-	@rm -f $@
-	@ln -sf $(CURDIR)/ $@
+	@if [ -h $@ ]; then rm -f $@; ln -sf $(CURDIR)/ $@; fi
 	@(cd $(DOCKER_MAIN); go get $(GO_OPTIONS))
 
 whichrelease:
@@ -75,4 +74,7 @@ fmt:
 	@gofmt -s -l -w .
 
 hack:
-	cd $(CURDIR)/buildbot && vagrant up
+	cd $(CURDIR)/hack && vagrant up
+
+ssh-dev:
+	cd $(CURDIR)/hack && vagrant ssh

hack/Vagrantfile

@@ -0,0 +1,36 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+BOX_NAME = "ubuntu"
+BOX_URI = "http://files.vagrantup.com/precise64.box"
+PPA_KEY = "E61D797F63561DC6"
+VM_IP = "192.168.33.21"
+USER = "vagrant"
+GOPATH = "/data/docker"
+DOCKER_PATH = "#{GOPATH}/src/github.com/dotcloud/docker"
+CFG_PATH = "#{DOCKER_PATH}/hack/environment"
+BUILDBOT_PATH = "/data/buildbot"
+
+Vagrant::Config.run do |config|
+  # Setup virtual machine box
+  config.vm.box = BOX_NAME
+  config.vm.box_url = BOX_URI
+  config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.."
+  config.vm.network :hostonly, VM_IP
+  # Stop if deployment has been done
+  config.vm.provision :shell, :inline => "[ ! -f /usr/bin/git ]"
+  # Touch for makefile
+  pkg_cmd = "touch #{DOCKER_PATH}; "
+  # Install docker dependencies
+  pkg_cmd << "export DEBIAN_FRONTEND=noninteractive; apt-get -qq update; " \
+    "apt-get install -q -y lxc bsdtar git golang make; " \
+    "chown -R #{USER}.#{USER} #{GOPATH}; " \
+    "install -m 0664 #{CFG_PATH}/bash_profile /home/#{USER}/.bash_profile"
+  config.vm.provision :shell, :inline => pkg_cmd
+  # Deploy buildbot CI
+  pkg_cmd = "apt-get install -q -y python-dev python-pip supervisor; " \
+    "pip install -r #{CFG_PATH}/requirements.txt; " \
+    "chown #{USER}.#{USER} /data; cd /data; " \
+    "#{CFG_PATH}/setup.sh #{USER} #{GOPATH} #{DOCKER_PATH} #{CFG_PATH} #{BUILDBOT_PATH}"
+  config.vm.provision :shell, :inline => pkg_cmd
+end

hack/environment/README.rst

@@ -0,0 +1,20 @@
+Buildbot
+========
+
+Buildbot is a continuous integration system designed to automate the
+build/test cycle. By automatically rebuilding and testing the tree each time
+something has changed, build problems are pinpointed quickly, before other
+developers are inconvenienced by the failure.
+
+When running 'make hack' at the docker root directory, it spawns a virtual
+machine in the background running a buildbot instance and adds a git
+post-commit hook that automatically run docker tests for you.
+
+You can check your buildbot instance at http://192.168.33.21:8010/waterfall
+
+
+Buildbot dependencies
+---------------------
+
+vagrant, virtualbox packages and python package requests
+

hack/environment/bash_profile

@@ -0,0 +1,19 @@
+# ~/.bash_profile : executed by the command interpreter for login shells.
+
+# if running bash
+if [ -n "$BASH_VERSION" ]; then
+    # include .bashrc if it exists
+    if [ -f "$HOME/.bashrc" ]; then
+        . "$HOME/.bashrc"
+    fi
+fi
+
+# set PATH so it includes user's private bin if it exists
+[ -d "$HOME/bin" ] && PATH="$HOME/bin:$PATH"
+
+docker=/data/docker/src/github.com/dotcloud/docker
+[ -d $docker ] && cd $docker
+
+export GOPATH=/data/docker
+export PATH=$PATH:$GOPATH/bin
+

hack/environment/master.cfg

@@ -13,8 +13,8 @@ TEST_USER = 'buildbot'  # Credential to authenticate build triggers
 TEST_PWD = 'docker'     # Credential to authenticate build triggers
 BUILDER_NAME = 'docker'
 BUILDPASSWORD = 'pass-docker'  # Credential to authenticate buildworkers
-DOCKER_PATH = '/data/docker'
-
+GOPATH = '/data/docker'
+DOCKER_PATH = '{0}/src/github.com/dotcloud/docker'.format(GOPATH)
 
 c = BuildmasterConfig = {}
 
@@ -28,10 +28,7 @@ c['slavePortnum'] = PORT_MASTER
 c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])]
 
 # Docker test command
-test_cmd = """(
-    cd {0}/..; rm -rf docker-tmp; git clone docker docker-tmp;
-    cd docker-tmp; make test; exit_status=$?;
-    cd ..; rm -rf docker-tmp; exit $exit_status)""".format(DOCKER_PATH)
+test_cmd = "GOPATH={0} make -C {1} test".format(GOPATH,DOCKER_PATH)
 
 # Builder
 factory = BuildFactory()

hack/environment/requirements.txt

@@ -0,0 +1,6 @@
+sqlalchemy<=0.7.9
+sqlalchemy-migrate>=0.7.2
+buildbot==0.8.7p1
+buildbot_slave==0.8.7p1
+nose==1.2.1
+requests==1.1.0

hack/environment/setup.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Setup of buildbot configuration. Package installation is being done by
+# Vagrantfile
+# Dependencies: buildbot, buildbot-slave, supervisor
+
+USER=$1
+GOPATH=$2
+DOCKER_PATH=$3
+CFG_PATH=$4
+BUILDBOT_PATH=$5
+SLAVE_NAME="buildworker"
+SLAVE_SOCKET="localhost:9989"
+BUILDBOT_PWD="pass-docker"
+IP=$(sed -nE 's/VM_IP = "(.+)"/\1/p' ${DOCKER_PATH}/hack/Vagrantfile)
+export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin"
+
+function run { su $USER -c "$1"; }
+
+# Exit if buildbot has already been installed
+[ -d "$BUILDBOT_PATH" ] && exit 0
+
+# Setup buildbot
+run "mkdir -p $BUILDBOT_PATH"
+cd $BUILDBOT_PATH
+run "buildbot create-master master"
+run "cp $CFG_PATH/master.cfg master"
+run "sed -i 's/localhost/$IP/' master/master.cfg"
+run "sed -i -E 's#(GOPATH = ).+#\1\"$GOPATH\"#' master/master.cfg"
+run "sed -i -E 's#(DOCKER_PATH = ).+#\1\"$DOCKER_PATH\"#' master/master.cfg"
+run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
+
+# Allow buildbot subprocesses (docker tests) to properly run in containers,
+# in particular with docker -u
+run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac"
+
+# Setup supervisor
+cp $CFG_PATH/buildbot.conf /etc/supervisor/conf.d/buildbot.conf
+sed -i -E "s/^chmod=0700.+/chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf
+kill -HUP $(pgrep -f "/usr/bin/python /usr/bin/supervisord")
+
+# Add git hook
+cp $CFG_PATH/post-commit $DOCKER_PATH/.git/hooks
+sed -i "s/localhost/$IP/" $DOCKER_PATH/.git/hooks/post-commit
+