diff --git a/utils/test/MultiTestRunner.py b/utils/test/MultiTestRunner.py index 53c1cdbb18..04bd74f65d 100755 --- a/utils/test/MultiTestRunner.py +++ b/utils/test/MultiTestRunner.py @@ -23,30 +23,22 @@ kTestFileExtensions = set(['.mi','.i','.c','.cpp','.m','.mm','.ll']) def getTests(inputs): for path in inputs: - # Always use absolte paths. - path = os.path.abspath(path) if not os.path.exists(path): print >>sys.stderr,"WARNING: Invalid test \"%s\""%(path,) continue - if os.path.isdir(path): - for dirpath,dirnames,filenames in os.walk(path): - dotTests = os.path.join(dirpath,'.tests') - if os.path.exists(dotTests): - for ln in open(dotTests): - if ln.strip(): - yield os.path.join(dirpath,ln.strip()) - else: - # FIXME: This doesn't belong here - if 'Output' in dirnames: - dirnames.remove('Output') - for f in filenames: - base,ext = os.path.splitext(f) - if ext in kTestFileExtensions: - yield os.path.join(dirpath,f) - else: + if not os.path.isdir(path): yield path + for dirpath,dirnames,filenames in os.walk(path): + # FIXME: This doesn't belong here + if 'Output' in dirnames: + dirnames.remove('Output') + for f in filenames: + base,ext = os.path.splitext(f) + if ext in kTestFileExtensions: + yield os.path.join(dirpath,f) + class TestingProgressDisplay: def __init__(self, opts, numTests, progressBar=None): self.opts = opts @@ -92,22 +84,21 @@ class TestingProgressDisplay: else: sys.stdout.write('\n') - extra = '' - if tr.code==TestStatus.Invalid: - extra = ' - (Invalid test)' - elif tr.failed(): - extra = ' - %s'%(TestStatus.getName(tr.code).upper(),) - print '%*d/%*d - %s%s'%(self.digits, index+1, self.digits, - self.numTests, tr.path, extra) + status = TestStatus.getName(tr.code).upper() + print '%s: %s (%*d of %*d)' % (status, tr.path, + self.digits, index+1, + self.digits, self.numTests) if tr.failed() and self.opts.showOutput: - TestRunner.cat(tr.testResults, sys.stdout) + print "%s TEST '%s' FAILED %s" % ('*'*20, tr.path, '*'*20) + print tr.output + print "*" * 20 class TestResult: - def __init__(self, path, code, testResults, elapsed): + def __init__(self, path, code, output, elapsed): self.path = path self.code = code - self.testResults = testResults + self.output = output self.elapsed = elapsed def failed(self): @@ -153,13 +144,8 @@ class Tester(threading.Thread): break self.runTest(item) - def runTest(self, (path,index)): - command = path + def runTest(self, (path, index)): base = TestRunner.getTestOutputBase('Output', path) - output = base + '.out' - testname = path - testresults = base + '.testresults' - TestRunner.mkdir_p(os.path.dirname(testresults)) numTests = len(self.provider.tests) digits = len(str(numTests)) code = None @@ -170,9 +156,8 @@ class Tester(threading.Thread): code = None else: startTime = time.time() - code = TestRunner.runOneTest(path, command, output, testname, - opts.clang, opts.clangcc, - output=open(testresults,'w')) + code, output = TestRunner.runOneTest(path, base, + opts.clang, opts.clangcc) elapsed = time.time() - startTime except KeyboardInterrupt: # This is a sad hack. Unfortunately subprocess goes @@ -180,8 +165,7 @@ class Tester(threading.Thread): print 'Ctrl-C detected, goodbye.' os.kill(0,9) - self.provider.setResult(index, TestResult(path, code, testresults, - elapsed)) + self.provider.setResult(index, TestResult(path, code, output, elapsed)) def detectCPUs(): """ diff --git a/utils/test/TestRunner.py b/utils/test/TestRunner.py index ddd543c30f..9e8b77e59b 100755 --- a/utils/test/TestRunner.py +++ b/utils/test/TestRunner.py @@ -57,54 +57,29 @@ def mkdir_p(path): if e.errno != errno.EEXIST: raise -def remove(path): - try: - os.remove(path) - except OSError: - pass - -def cat(path, output): - f = open(path) - output.writelines(f) - f.close() - -def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC, - output=sys.stdout): - OUTPUT = os.path.abspath(OUTPUT) +import StringIO +def runOneTest(testPath, tmpBase, clang, clangcc): + # Make paths absolute. + tmpBase = os.path.abspath(tmpBase) + testPath = os.path.abspath(testPath) # Create the output directory if it does not already exist. - mkdir_p(os.path.dirname(OUTPUT)) - scriptFile = FILENAME - - # Verify the script contains a run line. - for ln in open(scriptFile): - if 'RUN:' in ln: - break - else: - print >>output, "******************** TEST '%s' HAS NO RUN LINE! ********************"%(TESTNAME,) - output.flush() - return TestStatus.Fail - - FILENAME = os.path.abspath(FILENAME) - SCRIPT = OUTPUT + '.script' + mkdir_p(os.path.dirname(tmpBase)) + script = tmpBase + '.script' if kSystemName == 'Windows': - SCRIPT += '.bat' - TEMPOUTPUT = OUTPUT + '.tmp' + script += '.bat' - substitutions = [('%s',SUBST), - ('%S',os.path.dirname(SUBST)), - ('%llvmgcc','llvm-gcc -emit-llvm -w'), - ('%llvmgxx','llvm-g++ -emit-llvm -w'), - ('%prcontext','prcontext.tcl'), - ('%t',TEMPOUTPUT), - (' clang ', ' ' + CLANG + ' '), - (' clang-cc ', ' ' + CLANGCC + ' ')] + substitutions = [('%s', testPath), + ('%S', os.path.dirname(testPath)), + ('%t', tmpBase + '.tmp'), + (' clang ', ' ' + clang + ' '), + (' clang-cc ', ' ' + clangcc + ' ')] # Collect the test lines from the script. scriptLines = [] xfailLines = [] - for ln in open(scriptFile): + for ln in open(testPath): if 'RUN:' in ln: # Isolate the command to run. index = ln.index('RUN:') @@ -117,6 +92,10 @@ def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC, # FIXME: Support something like END, in case we need to process large # files. + + # Verify the script contains a run line. + if not scriptLines: + return (TestStatus.Fail, "Test has no run line!") # Apply substitutions to the script. def processLine(ln): @@ -133,19 +112,15 @@ def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC, ln = scriptLines[i] if not ln.endswith('&&'): - print >>output, "MISSING \'&&\': %s" % ln - print >>output, "FOLLOWED BY : %s" % scriptLines[i + 1] - return TestStatus.Fail + return (TestStatus.Fail, + "MISSING \'&&\': %s\n" + + "FOLLOWED BY : %s\n" % (ln,scriptLines[i + 1])) # Strip off '&&' scriptLines[i] = ln[:-2] - if xfailLines: - print >>output, "XFAILED '%s':"%(TESTNAME,) - output.writelines(xfailLines) - # Write script file - f = open(SCRIPT,'w') + f = open(script,'w') if kSystemName == 'Windows': f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(scriptLines)) else: @@ -153,52 +128,53 @@ def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC, f.write('\n') f.close() - outputFile = open(OUTPUT,'w') p = None try: if kSystemName == 'Windows': - command = ['cmd','/c', SCRIPT] + command = ['cmd','/c', script] else: - command = ['/bin/sh', SCRIPT] + command = ['/bin/sh', script] p = subprocess.Popen(command, - cwd=os.path.dirname(FILENAME), + cwd=os.path.dirname(testPath), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=kChildEnv) out,err = p.communicate() - outputFile.write(out) - outputFile.write(err) - SCRIPT_STATUS = p.wait() + exitCode = p.wait() # Detect Ctrl-C in subprocess. - if SCRIPT_STATUS == -signal.SIGINT: + if exitCode == -signal.SIGINT: raise KeyboardInterrupt except KeyboardInterrupt: raise - outputFile.close() if xfailLines: - SCRIPT_STATUS = not SCRIPT_STATUS - - if SCRIPT_STATUS: - print >>output, "******************** TEST '%s' FAILED! ********************"%(TESTNAME,) - print >>output, "Command: " - output.writelines(scriptLines) - print >>output, "Incorrect Output:" - cat(OUTPUT, output) - print >>output, "******************** TEST '%s' FAILED! ********************"%(TESTNAME,) - output.flush() - if xfailLines: - return TestStatus.XPass - else: - return TestStatus.Fail - - if xfailLines: - return TestStatus.XFail + ok = exitCode != 0 + status = (TestStatus.XPass, TestStatus.XFail)[ok] else: - return TestStatus.Pass + ok = exitCode == 0 + status = (TestStatus.Fail, TestStatus.Pass)[ok] + + if ok: + return (status,'') + + output = StringIO.StringIO() + print >>output, "Script:" + print >>output, "--" + print >>output, '\n'.join(scriptLines) + print >>output, "--" + print >>output, "Exit Code: %r" % exitCode + print >>output, "Command Output (stdout):" + print >>output, "--" + output.write(out) + print >>output, "--" + print >>output, "Command Output (stderr):" + print >>output, "--" + output.write(err) + print >>output, "--" + return (status, output.getvalue()) def capture(args): p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -304,14 +280,17 @@ def main(): opts.clangcc = inferClangCC(opts.clang) for path in args: - command = path - output = getTestOutputBase('Output', path) + '.out' - testname = path + base = getTestOutputBase('Output', path) + '.out' - res = runOneTest(path, command, output, testname, - opts.clang, opts.clangcc) + status,output = runOneTest(path, base, opts.clang, opts.clangcc) + print '%s: %s' % (TestStatus.getName(status).upper(), path) + if status == TestStatus.Fail or status == TestStatus.XPass: + print "%s TEST '%s' FAILED %s" % ('*'*20, path, '*'*20) + sys.stdout.write(output) + print "*" * 20 + sys.exit(1) - sys.exit(res == TestStatus.Fail or res == TestStatus.XPass) + sys.exit(0) if __name__=='__main__': main()