From 4a099cdd199fbcd60d69262b0ea88778fd70fc3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 8 Apr 2015 17:28:41 +0300 Subject: [PATCH 1/2] Fix tests/parallel_test_core.py to be multiprocess-compatible. --- tests/parallel_test_core.py | 43 ++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/parallel_test_core.py b/tests/parallel_test_core.py index b53b8fef8..d8a512ace 100755 --- a/tests/parallel_test_core.py +++ b/tests/parallel_test_core.py @@ -8,11 +8,6 @@ from runner import test_modes, PYTHON, path_from_root optimal_order = ['asm3i', 'asm1i', 'asm2nn', 'asm3', 'asm2', 'asm2g', 'asm2f', 'asm1', 'default'] assert set(optimal_order) == set(test_modes), 'need to update the list of slowest modes' -# clean up previous output -for mode in optimal_order: - if os.path.exists(mode + '.err'): - os.unlink(mode + '.err') - # set up a background thread to report progress class Watcher(threading.Thread): stop = False @@ -29,9 +24,6 @@ class Watcher(threading.Thread): print >> sys.stderr, '[parallel_test_copy.py watcher] total output: %d' % total time.sleep(1) -watcher = Watcher() -watcher.start() - # run tests for one mode def run_mode(mode): print '<< running %s >>' % mode @@ -39,18 +31,29 @@ def run_mode(mode): proc.communicate() print '<< %s finished >>' % mode -# run all modes -cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) -pool = multiprocessing.Pool(processes=cores) -filenames = pool.map(run_mode, optimal_order, chunksize=1) +def main(): + # clean up previous output + for mode in optimal_order: + if os.path.exists(mode + '.err'): + os.unlink(mode + '.err') -# quit watcher -Watcher.stop = True + watcher = Watcher() + watcher.start() -# emit all outputs -for mode in optimal_order: - print >> sys.stderr, '=== %s ===' % mode - if os.path.exists(mode + '.err'): - print >> sys.stderr, open(mode + '.err').read() - print >> sys.stderr + # run all modes + cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) + pool = multiprocessing.Pool(processes=cores) + filenames = pool.map(run_mode, optimal_order, chunksize=1) + # quit watcher + Watcher.stop = True + + # emit all outputs + for mode in optimal_order: + print >> sys.stderr, '=== %s ===' % mode + if os.path.exists(mode + '.err'): + print >> sys.stderr, open(mode + '.err').read() + print >> sys.stderr + +if __name__ == '__main__': + sys.exit(main()) From 89dd76c5dc67fe5bd56c0265edf2fc97438583d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Wed, 8 Apr 2015 17:35:35 +0300 Subject: [PATCH 2/2] Add support for passing command line parameters to test runs spawned from parallel test runner. --- tests/parallel_test_core.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/parallel_test_core.py b/tests/parallel_test_core.py index d8a512ace..def5f694c 100755 --- a/tests/parallel_test_core.py +++ b/tests/parallel_test_core.py @@ -25,13 +25,14 @@ class Watcher(threading.Thread): time.sleep(1) # run tests for one mode -def run_mode(mode): +def run_mode(args): + mode = args[0] print '<< running %s >>' % mode - proc = subprocess.Popen([PYTHON, path_from_root('tests', 'runner.py'), mode], stdout=open(mode + '.out', 'w'), stderr=open(mode + '.err', 'w')) + proc = subprocess.Popen([PYTHON, path_from_root('tests', 'runner.py')] + args, stdout=open(mode + '.out', 'w'), stderr=open(mode + '.err', 'w')) proc.communicate() print '<< %s finished >>' % mode -def main(): +def main(): # clean up previous output for mode in optimal_order: if os.path.exists(mode + '.err'): @@ -43,7 +44,8 @@ def main(): # run all modes cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) pool = multiprocessing.Pool(processes=cores) - filenames = pool.map(run_mode, optimal_order, chunksize=1) + args = [[x] + sys.argv[1:] for x in optimal_order] + filenames = pool.map(run_mode, args, chunksize=1) # quit watcher Watcher.stop = True