Merge pull request #3322 from juj/parallel_test_runner

Parallel test runner

tests/parallel_test_core.py

@@ -8,11 +8,6 @@ from runner import test_modes, PYTHON, path_from_root
 optimal_order = ['asm3i', 'asm1i', 'asm2nn', 'asm3', 'asm2', 'asm2g', 'asm2f', 'asm1', 'default']
 assert set(optimal_order) == set(test_modes), 'need to update the list of slowest modes'
 
-# clean up previous output
-for mode in optimal_order:
-  if os.path.exists(mode + '.err'):
-    os.unlink(mode + '.err')
-
 # set up a background thread to report progress
 class Watcher(threading.Thread):
   stop = False
@@ -29,28 +24,38 @@ class Watcher(threading.Thread):
         print >> sys.stderr, '[parallel_test_copy.py watcher] total output: %d' % total
       time.sleep(1)
 
-watcher = Watcher()
-watcher.start()
-
 # run tests for one mode
-def run_mode(mode):
+def run_mode(args):
+  mode = args[0]
   print '<< running %s >>' % mode
-  proc = subprocess.Popen([PYTHON, path_from_root('tests', 'runner.py'), mode], stdout=open(mode + '.out', 'w'), stderr=open(mode + '.err', 'w'))
+  proc = subprocess.Popen([PYTHON, path_from_root('tests', 'runner.py')] + args, stdout=open(mode + '.out', 'w'), stderr=open(mode + '.err', 'w'))
   proc.communicate()
   print '<< %s finished >>' % mode
 
-# run all modes
-cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count())
-pool = multiprocessing.Pool(processes=cores)
-filenames = pool.map(run_mode, optimal_order, chunksize=1)
+def main():
+  # clean up previous output
+  for mode in optimal_order:
+    if os.path.exists(mode + '.err'):
+      os.unlink(mode + '.err')
 
-# quit watcher
-Watcher.stop = True
+  watcher = Watcher()
+  watcher.start()
 
-# emit all outputs
-for mode in optimal_order:
-  print >> sys.stderr, '=== %s ===' % mode
-  if os.path.exists(mode + '.err'):
-    print >> sys.stderr, open(mode + '.err').read()
-  print >> sys.stderr
+  # run all modes
+  cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count())
+  pool = multiprocessing.Pool(processes=cores)
+  args = [[x] + sys.argv[1:] for x in optimal_order]
+  filenames = pool.map(run_mode, args, chunksize=1)
 
+  # quit watcher
+  Watcher.stop = True
+
+  # emit all outputs
+  for mode in optimal_order:
+    print >> sys.stderr, '=== %s ===' % mode
+    if os.path.exists(mode + '.err'):
+      print >> sys.stderr, open(mode + '.err').read()
+    print >> sys.stderr
+
+if __name__ == '__main__':
+  sys.exit(main())