From 81ce891cbde21db64ec24571b14d46f7bcebb26d Mon Sep 17 00:00:00 2001 From: "ilevy@chromium.org" Date: Mon, 10 Jun 2013 17:53:25 +0000 Subject: [PATCH] Port remaining android buildbot code into python - Run envsetup.sh and retrieve environment rather than wrapping commands in a bash shell. Consolidate remaining PATH code. This also cleans up the preamble output. - Port android landmine code to python. - Simplified testing command layout in bb_run_bot. - Change bb_run_bot_test to run in parallel. - Fixed bug where dbg clobber affects release bot using same checkout - Made step names more similar to other chromium bots. - Consolidated factory-prop and build-prop parsing code. BUG=176126 NOTRY=True Review URL: https://chromiumcodereview.appspot.com/16688002 git-svn-id: http://src.chromium.org/svn/trunk/src/build@205244 4ff67af0-8c30-449e-8e8b-ad334ec8d88c --- android/buildbot/bb_device_steps.py | 11 -- android/buildbot/bb_host_steps.py | 57 +++---- android/buildbot/bb_run_bot.py | 190 +++++++++++----------- android/buildbot/bb_utils.py | 19 ++- android/buildbot/buildbot_functions.sh | 92 ----------- android/buildbot/env_to_json.py | 11 ++ android/buildbot/tests/bb_run_bot_test.py | 12 +- 7 files changed, 151 insertions(+), 241 deletions(-) delete mode 100755 android/buildbot/buildbot_functions.sh create mode 100755 android/buildbot/env_to_json.py diff --git a/android/buildbot/bb_device_steps.py b/android/buildbot/bb_device_steps.py index 15df49041..621dd0869 100755 --- a/android/buildbot/bb_device_steps.py +++ b/android/buildbot/bb_device_steps.py @@ -306,17 +306,6 @@ def main(argv): setattr(options, 'target', options.factory_properties.get('target', 'Debug')) - # Add adb binary and chromium-source platform-tools to tip of PATH variable. - android_paths = [os.path.join(constants.ANDROID_SDK_ROOT, 'platform-tools')] - - # Bots checkout chrome in /b/build/slave//build/src - build_internal_android = os.path.abspath(os.path.join( - CHROME_SRC, '..', '..', '..', '..', '..', 'build_internal', 'scripts', - 'slave', 'android')) - if os.path.exists(build_internal_android): - android_paths.insert(0, build_internal_android) - os.environ['PATH'] = os.pathsep.join(android_paths + [os.environ['PATH']]) - MainTestWrapper(options) diff --git a/android/buildbot/bb_host_steps.py b/android/buildbot/bb_host_steps.py index b4cea41bf..7539c4290 100755 --- a/android/buildbot/bb_host_steps.py +++ b/android/buildbot/bb_host_steps.py @@ -3,7 +3,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import json import os import sys @@ -27,12 +26,24 @@ def SrcPath(*path): def CheckWebViewLicenses(): - buildbot_report.PrintNamedStep('Check licenses for WebView') + buildbot_report.PrintNamedStep('check_licenses') RunCmd([SrcPath('android_webview', 'tools', 'webview_licenses.py'), 'scan'], warning_code=1) -def RunHooks(): +def RunHooks(build_type): + RunCmd([SrcPath('build', 'landmines.py')]) + build_path = SrcPath('out', build_type) + landmine_path = os.path.join(build_path, '.landmines_triggered') + clobber_env = os.environ.get('BUILDBOT_CLOBBER') + if clobber_env or os.path.isfile(landmine_path): + buildbot_report.PrintNamedStep('Clobber') + if not clobber_env: + print 'Clobbering due to triggered landmines:' + with open(landmine_path) as f: + print f.read() + RunCmd(['rm', '-rf', build_path]) + buildbot_report.PrintNamedStep('runhooks') RunCmd(['gclient', 'runhooks'], halt_on_failure=True) @@ -42,7 +53,7 @@ def Compile(build_type, args, experimental=False): '--build-tool=ninja', '--compiler=goma', '--target=%s' % build_type, - '--goma-dir=%s' % os.path.join(bb_utils.BB_BUILD_DIR, 'goma')] + '--goma-dir=%s' % bb_utils.GOMA_DIR] if experimental: for compile_target in args: buildbot_report.PrintNamedStep('Experimental Compile %s' % compile_target) @@ -52,23 +63,21 @@ def Compile(build_type, args, experimental=False): RunCmd(cmd + ['--build-args=%s' % ' '.join(args)], halt_on_failure=True) -def ZipBuild(factory_properties, build_properties): - buildbot_report.PrintNamedStep('Zip build') - RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'zip_build.py'), - '--src-dir', constants.DIR_SOURCE_ROOT, - '--build-dir', SrcPath('out'), - '--exclude-files', 'lib.target,gen,android_webview,jingle_unittests', - '--factory-properties', json.dumps(factory_properties), - '--build-properties', json.dumps(build_properties)]) +def ZipBuild(properties): + buildbot_report.PrintNamedStep('zip_build') + RunCmd([ + os.path.join(SLAVE_SCRIPTS_DIR, 'zip_build.py'), + '--src-dir', constants.DIR_SOURCE_ROOT, + '--build-dir', SrcPath('out'), + '--exclude-files', 'lib.target,gen,android_webview,jingle_unittests'] + + properties) -def ExtractBuild(factory_properties, build_properties): - buildbot_report.PrintNamedStep('Download and extract build') +def ExtractBuild(properties): + buildbot_report.PrintNamedStep('extract_build') RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'extract_build.py'), '--build-dir', SrcPath('build'), - '--build-output-dir', SrcPath('out'), - '--factory-properties', json.dumps(factory_properties), - '--build-properties', json.dumps(build_properties)], + '--build-output-dir', SrcPath('out')] + properties, warning_code=1) @@ -83,10 +92,6 @@ def FindBugs(is_release): 'run_findbugs_plugin_tests.py')] + build_type) -def UpdateClang(): - RunCmd([SrcPath('tools', 'clang', 'scripts', 'update.sh')]) - - def main(argv): parser = bb_utils.GetParser() parser.add_option('--host-tests', help='Comma separated list of host tests.') @@ -100,8 +105,6 @@ def main(argv): help='Indicate whether the build should be zipped.') parser.add_option('--extract-build', action='store_true', help='Indicate whether a build should be downloaded.') - parser.add_option('--update-clang', action='store_true', - help='Download or build the ASan runtime library.') options, args = parser.parse_args(argv[1:]) if args: @@ -119,18 +122,16 @@ def main(argv): if options.compile: if 'check_webview_licenses' in host_tests: CheckWebViewLicenses() - RunHooks() + RunHooks(build_type) Compile(build_type, options.build_args.split(',')) if options.experimental: Compile(build_type, EXPERIMENTAL_TARGETS, True) if 'findbugs' in host_tests: FindBugs(build_type == 'Release') if options.zip_build: - ZipBuild(options.factory_properties, options.build_properties) - if options.update_clang: - UpdateClang() + ZipBuild(bb_utils.EncodeProperties(options)) if options.extract_build: - ExtractBuild(options.factory_properties, options.build_properties) + ExtractBuild(bb_utils.EncodeProperties(options)) if __name__ == '__main__': diff --git a/android/buildbot/bb_run_bot.py b/android/buildbot/bb_run_bot.py index 535bc41e7..bf831e02d 100755 --- a/android/buildbot/bb_run_bot.py +++ b/android/buildbot/bb_run_bot.py @@ -7,29 +7,65 @@ import collections import copy import json -import optparse import os import pipes +import re import subprocess import sys import bb_utils -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -from pylib import buildbot_report - -CHROME_SRC = os.path.abspath( - os.path.join(os.path.dirname(__file__), '..', '..', '..')) - -GLOBAL_SLAVE_PROPS = {} - BotConfig = collections.namedtuple( - 'BotConfig', ['bot_id', 'host_opts', 'test_obj', 'slave_props']) -TestConfig = collections.namedtuple('Tests', ['tests', 'extra_args']) -Command = collections.namedtuple( - 'Command', ['step_name', 'command', 'testing_cmd']) + 'BotConfig', ['bot_id', 'host_obj', 'test_obj']) -CommandToString = bb_utils.CommandToString +HostConfig = collections.namedtuple( + 'HostConfig', ['host_step_args', 'extra_gyp_defines', 'target_arch']) + +TestConfig = collections.namedtuple('Tests', ['tests', 'extra_args']) + + +def DictDiff(d1, d2): + diff = [] + for key in sorted(set(d1.keys() + d2.keys())): + if key in d1 and d1[key] != d2.get(key): + diff.append('- %s=%s' % (key, pipes.quote(d1[key]))) + if key in d2 and d2[key] != d1.get(key): + diff.append('+ %s=%s' % (key, pipes.quote(d2[key]))) + return '\n'.join(diff) + + +def GetEnvironment(host_obj): + init_env = dict(os.environ) + init_env['GYP_GENERATORS'] = 'ninja' + init_env['GOMA_DIR'] = bb_utils.GOMA_DIR + envsetup_cmd = '. build/android/envsetup.sh' + if host_obj.target_arch: + envsetup_cmd += ' --target_arch=%s' % host_obj.target_arch + print 'Running %s' % envsetup_cmd + proc = subprocess.Popen(['bash', '-exc', + envsetup_cmd + ' >&2; python build/android/buildbot/env_to_json.py'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=bb_utils.CHROME_SRC, env=init_env) + json_env, envsetup_output = proc.communicate() + if proc.returncode != 0: + print 'FATAL Failure in envsetup.' + print envsetup_output + sys.exit(1) + env = json.loads(json_env) + env['GYP_DEFINES'] = env.get('GYP_DEFINES', '') + ' fastbuild=1' + extra_gyp = host_obj.extra_gyp_defines + if extra_gyp: + env['GYP_DEFINES'] += ' %s' % extra_gyp + if re.search('(asan|clang)=1', extra_gyp): + env.pop('CXX_target', None) + + # Bots checkout chrome in /b/build/slave//build/src + build_internal_android = os.path.abspath(os.path.join( + bb_utils.CHROME_SRC, '..', '..', '..', '..', '..', 'build_internal', + 'scripts', 'slave', 'android')) + if os.path.exists(build_internal_android): + env['PATH'] = os.pathsep.join([build_internal_android, env['PATH']]) + return env def GetCommands(options, bot_config): @@ -41,32 +77,9 @@ def GetCommands(options, bot_config): Returns: list of Command objects. """ - slave_props = dict(GLOBAL_SLAVE_PROPS) - if bot_config.slave_props: - slave_props.update(bot_config.slave_props) - - slave_properties = json.dumps(slave_props) - property_args = [ - '--factory-properties=%s' % json.dumps(options.factory_properties), - '--build-properties=%s' % json.dumps(options.build_properties), - '--slave-properties=%s' % slave_properties] - - commands = [] - def WrapWithBash(command): - """Wrap a bash command string with envsetup scripts.""" - return ['bash', '-exc', '; '.join([ - '. build/android/buildbot/buildbot_functions.sh', - 'bb_baseline_setup %s --slave-properties=%s' % ( - CHROME_SRC, pipes.quote(slave_properties)), - command]) - ] - - if bot_config.host_opts: - host_cmd = (['build/android/buildbot/bb_host_steps.py'] + - bot_config.host_opts + property_args) - commands.append(Command( - 'Host steps', - WrapWithBash(' '.join(map(pipes.quote, host_cmd))), host_cmd)) + property_args = bb_utils.EncodeProperties(options) + commands = [['build/android/buildbot/bb_host_steps.py'] + + bot_config.host_obj.host_step_args + property_args] test_obj = bot_config.test_obj if test_obj: @@ -76,10 +89,7 @@ def GetCommands(options, bot_config): run_test_cmd.extend(['-f', test]) if test_obj.extra_args: run_test_cmd.extend(test_obj.extra_args) - commands.append(Command( - 'Run tests', - WrapWithBash(' '.join(map(pipes.quote, run_test_cmd))), run_test_cmd)) - + commands.append(run_test_cmd) return commands @@ -90,45 +100,47 @@ def GetBotStepMap(): std_test_opts = ['--extract-build'] std_tests = ['ui', 'unit'] flakiness_server = '--upload-to-flakiness-server' - extra_gyp = 'extra_gyp_defines' - def B(bot_id, bash_funs, test_obj=None, slave_props=None): - return BotConfig(bot_id, bash_funs, test_obj, slave_props) + def B(bot_id, host_object, test_object=None): + return BotConfig(bot_id, host_object, test_object) def T(tests, extra_args=None): return TestConfig(tests, extra_args) + def H(host_step_args, extra_gyp=None, target_arch=None): + return HostConfig(host_step_args, extra_gyp, target_arch) + bot_configs = [ # Main builders - B('main-builder-dbg', std_build_opts + std_host_tests), - B('main-builder-rel', std_build_opts), - B('main-clang-builder', compile_opt, slave_props={extra_gyp: 'clang=1'}), - B('main-clobber', compile_opt), - B('main-tests', std_test_opts, T(std_tests, [flakiness_server])), + B('main-builder-dbg', H(std_build_opts + std_host_tests)), + B('main-builder-rel', H(std_build_opts)), + B('main-clang-builder', H(compile_opt, 'clang=1')), + B('main-clobber', H(compile_opt)), + B('main-tests', H(std_test_opts), T(std_tests, [flakiness_server])), # Other waterfalls - B('asan-builder-tests', compile_opt + ['--update-clang'], - T(std_tests, ['--asan']), {extra_gyp: 'asan=1'}), - B('chromedriver-fyi-tests-dbg', std_test_opts, + B('asan-builder-tests', H(compile_opt, 'asan=1'), + T(std_tests, ['--asan'])), + B('chromedriver-fyi-tests-dbg', H(std_test_opts), T(['chromedriver'], ['--install=ChromiumTestShell'])), B('fyi-builder-dbg', - std_build_opts + std_host_tests + ['--experimental']), - B('fyi-builder-rel', std_build_opts + ['--experimental']), - B('fyi-tests-dbg-ics-gn', compile_opt + [ '--experimental'], + H(std_build_opts + std_host_tests + ['--experimental'])), + B('fyi-builder-rel', H(std_build_opts + ['--experimental'])), + B('fyi-tests-dbg-ics-gn', H(compile_opt + [ '--experimental']), T(std_tests, ['--experimental', flakiness_server])), - B('fyi-tests', std_test_opts, + B('fyi-tests', H(std_test_opts), T(std_tests, ['--experimental', flakiness_server])), - B('fyi-component-builder-tests-dbg', compile_opt, - T(std_tests, ['--experimental', flakiness_server]), - {extra_gyp: 'component=shared_library'}), - B('perf-tests-rel', std_test_opts, T([], ['--install=ContentShell'])), - B('webkit-latest-webkit-tests', std_test_opts, + B('fyi-component-builder-tests-dbg', + H(compile_opt, 'component=shared_library'), + T(std_tests, ['--experimental', flakiness_server])), + B('perf-tests-rel', H(std_test_opts), T([], ['--install=ContentShell'])), + B('webkit-latest-webkit-tests', H(std_test_opts), T(['webkit_layout', 'webkit'])), - B('webkit-latest-contentshell', compile_opt, T(['webkit_layout'])), - B('builder-unit-tests', compile_opt, T(['unit'])), + B('webkit-latest-contentshell', H(compile_opt), T(['webkit_layout'])), + B('builder-unit-tests', H(compile_opt), T(['unit'])), # Generic builder config (for substring match). - B('builder', std_build_opts), + B('builder', H(std_build_opts)), ] bot_map = dict((config.bot_id, config) for config in bot_configs) @@ -160,19 +172,9 @@ def GetBotStepMap(): def main(argv): - parser = optparse.OptionParser() - - def ConvertJson(option, _, value, parser): - setattr(parser.values, option.dest, json.loads(value)) - - parser.add_option('--build-properties', action='callback', - callback=ConvertJson, type='string', default={}, - help='build properties in JSON format') - parser.add_option('--factory-properties', action='callback', - callback=ConvertJson, type='string', default={}, - help='factory properties in JSON format') + parser = bb_utils.GetParser() parser.add_option('--bot-id', help='Specify bot id directly.') - parser.add_option('--TESTING', action='store_true', + parser.add_option('--testing', action='store_true', help='For testing: print, but do not run commands') options, args = parser.parse_args(argv[1:]) if args: @@ -200,26 +202,20 @@ def main(argv): print 'Using config:', bot_config - command_objs = GetCommands(options, bot_config) - for command_obj in command_objs: - print 'Will run:', CommandToString(command_obj.command) + commands = GetCommands(options, bot_config) + for command in commands: + print 'Will run: ', bb_utils.CommandToString(command) - for command_obj in command_objs: - if command_obj.step_name: - buildbot_report.PrintNamedStep(command_obj.step_name) - command = command_obj.command - print CommandToString(command) + env = GetEnvironment(bot_config.host_obj) + print 'Environment changes:' + print DictDiff(dict(os.environ), env) + + for command in commands: + print bb_utils.CommandToString(command) sys.stdout.flush() - env = None - if options.TESTING: - if not command_obj.testing_cmd: - continue - return_code = subprocess.call( - command_obj.testing_cmd, - cwd=CHROME_SRC, - env=dict(os.environ, BUILDBOT_TESTING='1')) - else: - return_code = subprocess.call(command, cwd=CHROME_SRC, env=env) + if options.testing: + env['BUILDBOT_TESTING'] = '1' + return_code = subprocess.call(command, cwd=bb_utils.CHROME_SRC, env=env) if return_code != 0: return return_code diff --git a/android/buildbot/bb_utils.py b/android/buildbot/bb_utils.py index bc9117a84..3f70caaac 100644 --- a/android/buildbot/bb_utils.py +++ b/android/buildbot/bb_utils.py @@ -11,7 +11,6 @@ import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..')) from pylib import buildbot_report -from pylib import constants TESTING = 'BUILDBOT_TESTING' in os.environ @@ -20,6 +19,10 @@ BB_BUILD_DIR = os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir)) +CHROME_SRC = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..')) + +GOMA_DIR = os.environ.get('GOMA_DIR', os.path.join(BB_BUILD_DIR, 'goma')) def CommandToString(command): """Returns quoted command that can be run in bash shell.""" @@ -36,8 +39,7 @@ def SpawnCmd(command): def wait(): return 0 return MockPopen() - - return subprocess.Popen(command, cwd=constants.DIR_SOURCE_ROOT) + return subprocess.Popen(command, cwd=CHROME_SRC) def RunCmd(command, flunk_on_failure=True, halt_on_failure=False, @@ -53,7 +55,8 @@ def RunCmd(command, flunk_on_failure=True, halt_on_failure=False, buildbot_report.PrintWarning() # Allow steps to have both halting (i.e. 1) and non-halting exit codes. if code != warning_code and halt_on_failure: - raise OSError() + print 'FATAL %d != %d' % (code, warning_code) + sys.exit(1) return code @@ -67,9 +70,9 @@ def GetParser(): parser.add_option('--factory-properties', action='callback', callback=ConvertJson, type='string', default={}, help='factory properties in JSON format') - parser.add_option('--slave-properties', action='callback', - callback=ConvertJson, type='string', default={}, - help='Properties set by slave script in JSON format') - return parser + +def EncodeProperties(options): + return ['--factory-properties=%s' % json.dumps(options.factory_properties), + '--build-properties=%s' % json.dumps(options.build_properties)] diff --git a/android/buildbot/buildbot_functions.sh b/android/buildbot/buildbot_functions.sh deleted file mode 100755 index f6f76290b..000000000 --- a/android/buildbot/buildbot_functions.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -# -# Bash functions used by buildbot annotator scripts for the android -# build of chromium. Executing this script should not perform actions -# other than setting variables and defining of functions. - -# Number of jobs on the compile line; e.g. make -j"${JOBS}" -JOBS="${JOBS:-4}" - -# Parse named arguments passed into the annotator script -# and assign them global variable names. -function bb_parse_args { - while [[ $1 ]]; do - case "$1" in - --factory-properties=*) - FACTORY_PROPERTIES="$(echo "$1" | sed 's/^[^=]*=//')" - BUILDTYPE=$(bb_get_json_prop "$FACTORY_PROPERTIES" target) - ;; - --build-properties=*) - BUILD_PROPERTIES="$(echo "$1" | sed 's/^[^=]*=//')" - ;; - --slave-properties=*) - SLAVE_PROPERTIES="$(echo "$1" | sed 's/^[^=]*=//')" - ;; - *) - echo "@@@STEP_WARNINGS@@@" - echo "Warning, unparsed input argument: '$1'" - ;; - esac - shift - done -} - -# Basic setup for all bots to run after a source tree checkout. -# Args: -# $1: source root. -# $2 and beyond: key value pairs which are parsed by bb_parse_args. -function bb_baseline_setup { - SRC_ROOT="$1" - # Remove SRC_ROOT param - shift - cd $SRC_ROOT - - bb_parse_args "$@" - - export GYP_GENERATORS=ninja - export GOMA_DIR=/b/build/goma - . build/android/envsetup.sh "" - - local extra_gyp_defines="$(bb_get_json_prop "$SLAVE_PROPERTIES" \ - extra_gyp_defines)" - export GYP_DEFINES+=" fastbuild=1 $extra_gyp_defines" - if echo $extra_gyp_defines | grep -qE 'clang|asan'; then - unset CXX_target - fi - - local build_path="${SRC_ROOT}/out/${BUILDTYPE}" - local landmines_triggered_path="$build_path/.landmines_triggered" - python "$SRC_ROOT/build/landmines.py" - - if [[ $BUILDBOT_CLOBBER || -f "$landmines_triggered_path" ]]; then - echo "@@@BUILD_STEP Clobber@@@" - - if [[ -z $BUILDBOT_CLOBBER ]]; then - echo "Clobbering due to triggered landmines: " - cat "$landmines_triggered_path" - else - # Also remove all the files under out/ on an explicit clobber - find "${SRC_ROOT}/out" -maxdepth 1 -type f -exec rm -f {} + - fi - - # Sdk key expires, delete android folder. - # crbug.com/145860 - rm -rf ~/.android - rm -rf "$build_path" - if [[ -e $build_path ]] ; then - echo "Clobber appeared to fail? $build_path still exists." - echo "@@@STEP_WARNINGS@@@" - fi - fi -} - -# Retrieve a packed json property using python -function bb_get_json_prop { - local JSON="$1" - local PROP="$2" - - python -c "import json; print json.loads('$JSON').get('$PROP', '')" -} diff --git a/android/buildbot/env_to_json.py b/android/buildbot/env_to_json.py new file mode 100755 index 000000000..f9a7a443d --- /dev/null +++ b/android/buildbot/env_to_json.py @@ -0,0 +1,11 @@ +#!/usr/bin/python +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Encode current environment into json. + +import json +import os + +print json.dumps(dict(os.environ)) diff --git a/android/buildbot/tests/bb_run_bot_test.py b/android/buildbot/tests/bb_run_bot_test.py index 968ebc234..3b028795b 100755 --- a/android/buildbot/tests/bb_run_bot_test.py +++ b/android/buildbot/tests/bb_run_bot_test.py @@ -14,14 +14,16 @@ import bb_run_bot def main(): code = 0 - for bot_id in bb_run_bot.GetBotStepMap(): - proc = subprocess.Popen( - [os.path.join(BUILDBOT_DIR, 'bb_run_bot.py'), '--bot-id', bot_id, - '--TESTING'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + procs = [ + (bot, subprocess.Popen( + [os.path.join(BUILDBOT_DIR, 'bb_run_bot.py'), '--bot-id', bot, + '--testing'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)) + for bot in bb_run_bot.GetBotStepMap()] + for bot, proc in procs: _, err = proc.communicate() code |= proc.returncode if proc.returncode != 0: - print 'Error running bb_run_bot with id="%s"' % bot_id, err + print 'Error running bb_run_bot with id="%s"' % bot, err return code