From bbaa975ec8ea44f7df4f96a910360abc43b4d5f7 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Tue, 9 Apr 2013 14:30:10 -0700 Subject: [PATCH] testing: Add buildbot VM --- Makefile | 3 ++ buildbot/README.rst | 20 ++++++++++++ buildbot/Vagrantfile | 28 ++++++++++++++++ buildbot/buildbot-cfg/buildbot-cfg.sh | 43 +++++++++++++++++++++++++ buildbot/buildbot-cfg/buildbot.conf | 18 +++++++++++ buildbot/buildbot-cfg/master.cfg | 46 +++++++++++++++++++++++++++ buildbot/buildbot-cfg/post-commit | 21 ++++++++++++ buildbot/buildbot.pp | 32 +++++++++++++++++++ buildbot/requirements.txt | 6 ++++ 9 files changed, 217 insertions(+) create mode 100644 buildbot/README.rst create mode 100644 buildbot/Vagrantfile create mode 100755 buildbot/buildbot-cfg/buildbot-cfg.sh create mode 100644 buildbot/buildbot-cfg/buildbot.conf create mode 100644 buildbot/buildbot-cfg/master.cfg create mode 100755 buildbot/buildbot-cfg/post-commit create mode 100644 buildbot/buildbot.pp create mode 100644 buildbot/requirements.txt diff --git a/Makefile b/Makefile index a6eb613830..94d8d44436 100644 --- a/Makefile +++ b/Makefile @@ -49,3 +49,6 @@ test: all fmt: @gofmt -s -l -w . + +hack: + @(cd $(CURDIR)/buildbot; vagrant up) diff --git a/buildbot/README.rst b/buildbot/README.rst new file mode 100644 index 0000000000..a52b9769ef --- /dev/null +++ b/buildbot/README.rst @@ -0,0 +1,20 @@ +Buildbot +======== + +Buildbot is a continuous integration system designed to automate the +build/test cycle. By automatically rebuilding and testing the tree each time +something has changed, build problems are pinpointed quickly, before other +developers are inconvenienced by the failure. + +When running 'make hack' at the docker root directory, it spawns a virtual +machine in the background running a buildbot instance and adds a git +post-commit hook that automatically run docker tests for you. + +You can check your buildbot instance at http://192.168.33.21:8010/waterfall + + +Buildbot dependencies +--------------------- + +vagrant, virtualbox packages and python package requests + diff --git a/buildbot/Vagrantfile b/buildbot/Vagrantfile new file mode 100644 index 0000000000..ea027f0666 --- /dev/null +++ b/buildbot/Vagrantfile @@ -0,0 +1,28 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +$BUILDBOT_IP = '192.168.33.21' + +def v10(config) + config.vm.box = "quantal64_3.5.0-25" + config.vm.box_url = "http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box" + config.vm.share_folder 'v-data', '/data/docker', File.dirname(__FILE__) + '/..' + config.vm.network :hostonly, $BUILDBOT_IP + + # Ensure puppet is installed on the instance + config.vm.provision :shell, :inline => 'apt-get -qq update; apt-get install -y puppet' + + config.vm.provision :puppet do |puppet| + puppet.manifests_path = '.' + puppet.manifest_file = 'buildbot.pp' + puppet.options = ['--templatedir','.'] + end +end + +Vagrant::VERSION < '1.1.0' and Vagrant::Config.run do |config| + v10(config) +end + +Vagrant::VERSION >= '1.1.0' and Vagrant.configure('1') do |config| + v10(config) +end diff --git a/buildbot/buildbot-cfg/buildbot-cfg.sh b/buildbot/buildbot-cfg/buildbot-cfg.sh new file mode 100755 index 0000000000..5e4e7432fd --- /dev/null +++ b/buildbot/buildbot-cfg/buildbot-cfg.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Auto setup of buildbot configuration. Package installation is being done +# on buildbot.pp +# Dependencies: buildbot, buildbot-slave, supervisor + +SLAVE_NAME='buildworker' +SLAVE_SOCKET='localhost:9989' +BUILDBOT_PWD='pass-docker' +USER='vagrant' +ROOT_PATH='/data/buildbot' +DOCKER_PATH='/data/docker' +BUILDBOT_CFG="$DOCKER_PATH/buildbot/buildbot-cfg" +IP=$(grep BUILDBOT_IP /data/docker/buildbot/Vagrantfile | awk -F "'" '{ print $2; }') + +function run { su $USER -c "$1"; } + +export PATH=/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin + +# Exit if buildbot has already been installed +[ -d "$ROOT_PATH" ] && exit 0 + +# Setup buildbot +run "mkdir -p ${ROOT_PATH}" +cd ${ROOT_PATH} +run "buildbot create-master master" +run "cp $BUILDBOT_CFG/master.cfg master" +run "sed -i 's/localhost/$IP/' master/master.cfg" +run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" + +# Allow buildbot subprocesses (docker tests) to properly run in containers, +# in particular with docker -u +run "sed -i 's/^umask = None/umask = 000/' ${ROOT_PATH}/slave/buildbot.tac" + +# Setup supervisor +cp $BUILDBOT_CFG/buildbot.conf /etc/supervisor/conf.d/buildbot.conf +sed -i "s/^chmod=0700.*0700./chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf +kill -HUP `pgrep -f "/usr/bin/python /usr/bin/supervisord"` + +# Add git hook +cp $BUILDBOT_CFG/post-commit $DOCKER_PATH/.git/hooks +sed -i "s/localhost/$IP/" $DOCKER_PATH/.git/hooks/post-commit + diff --git a/buildbot/buildbot-cfg/buildbot.conf b/buildbot/buildbot-cfg/buildbot.conf new file mode 100644 index 0000000000..b162f4e7c7 --- /dev/null +++ b/buildbot/buildbot-cfg/buildbot.conf @@ -0,0 +1,18 @@ +[program:buildmaster] +command=su vagrant -c "buildbot start master" +directory=/data/buildbot +chown= root:root +redirect_stderr=true +stdout_logfile=/var/log/supervisor/buildbot-master.log +stderr_logfile=/var/log/supervisor/buildbot-master.log + +[program:buildworker] +command=buildslave start slave +directory=/data/buildbot +chown= root:root +redirect_stderr=true +stdout_logfile=/var/log/supervisor/buildbot-slave.log +stderr_logfile=/var/log/supervisor/buildbot-slave.log + +[group:buildbot] +programs=buildmaster,buildworker diff --git a/buildbot/buildbot-cfg/master.cfg b/buildbot/buildbot-cfg/master.cfg new file mode 100644 index 0000000000..c786e418ed --- /dev/null +++ b/buildbot/buildbot-cfg/master.cfg @@ -0,0 +1,46 @@ +import os +from buildbot.buildslave import BuildSlave +from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.config import BuilderConfig +from buildbot.process.factory import BuildFactory +from buildbot.steps.shell import ShellCommand +from buildbot.status import html +from buildbot.status.web import authz, auth + +PORT_WEB = 8010 # Buildbot webserver port +PORT_MASTER = 9989 # Port where buildbot master listen buildworkers +TEST_USER = 'buildbot' # Credential to authenticate build triggers +TEST_PWD = 'docker' # Credential to authenticate build triggers +BUILDER_NAME = 'docker' +BUILDPASSWORD = 'pass-docker' # Credential to authenticate buildworkers +DOCKER_PATH = '/data/docker' + + +c = BuildmasterConfig = {} + +c['title'] = "Docker" +c['titleURL'] = "waterfall" +c['buildbotURL'] = "http://localhost:{0}/".format(PORT_WEB) +c['db'] = {'db_url':"sqlite:///state.sqlite"} +c['slaves'] = [BuildSlave('buildworker', BUILDPASSWORD)] +c['slavePortnum'] = PORT_MASTER + +c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])] + +# Docker test command +test_cmd = """( + cd {0}/..; rm -rf docker-tmp; git clone docker docker-tmp; + cd docker-tmp; make test; exit_status=$?; + cd ..; rm -rf docker-tmp; exit $exit_status)""".format(DOCKER_PATH) + +# Builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='Docker',logEnviron=False, + usePTY=True,command=test_cmd)) +c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], + factory=factory)] + +# Status +authz_cfg=authz.Authz(auth=auth.BasicAuth([(TEST_USER,TEST_PWD)]), + forceBuild='auth') +c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)] diff --git a/buildbot/buildbot-cfg/post-commit b/buildbot/buildbot-cfg/post-commit new file mode 100755 index 0000000000..8c5a06bf38 --- /dev/null +++ b/buildbot/buildbot-cfg/post-commit @@ -0,0 +1,21 @@ +#!/usr/bin/python + +'''Trigger buildbot docker test build + + post-commit git hook designed to automatically trigger buildbot on + the provided vagrant docker VM.''' + +import requests + +USERNAME = 'buildbot' +PASSWORD = 'docker' +BASE_URL = 'http://localhost:8010' +path = lambda s: BASE_URL + '/' + s + +try: + session = requests.session() + session.post(path('login'),data={'username':USERNAME,'passwd':PASSWORD}) + session.post(path('builders/docker/force'), + data={'forcescheduler':'trigger','reason':'Test commit'}) +except: + pass diff --git a/buildbot/buildbot.pp b/buildbot/buildbot.pp new file mode 100644 index 0000000000..8109cdc2a0 --- /dev/null +++ b/buildbot/buildbot.pp @@ -0,0 +1,32 @@ +node default { + $USER = 'vagrant' + $ROOT_PATH = '/data/buildbot' + $DOCKER_PATH = '/data/docker' + + exec {'apt_update': command => '/usr/bin/apt-get update' } + Package { require => Exec['apt_update'] } + group {'puppet': ensure => 'present'} + + # Install dependencies + Package { ensure => 'installed' } + package { ['python-dev','python-pip','supervisor','lxc','bsdtar','git','golang']: } + + file{[ '/data' ]: + owner => $USER, group => $USER, ensure => 'directory' } + + file {'/var/tmp/requirements.txt': + content => template('requirements.txt') } + + exec {'requirements': + require => [ Package['python-dev'], Package['python-pip'], + File['/var/tmp/requirements.txt'] ], + cwd => '/var/tmp', + command => "/bin/sh -c '(/usr/bin/pip install -r requirements.txt; + rm /var/tmp/requirements.txt)'" } + + exec {'buildbot-cfg-sh': + require => [ Package['supervisor'], Exec['requirements']], + path => '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin', + cwd => '/data', + command => "$DOCKER_PATH/buildbot/buildbot-cfg/buildbot-cfg.sh" } +} diff --git a/buildbot/requirements.txt b/buildbot/requirements.txt new file mode 100644 index 0000000000..0e451b017d --- /dev/null +++ b/buildbot/requirements.txt @@ -0,0 +1,6 @@ +sqlalchemy<=0.7.9 +sqlalchemy-migrate>=0.7.2 +buildbot==0.8.7p1 +buildbot_slave==0.8.7p1 +nose==1.2.1 +requests==1.1.0