MultiTestRunner: Cleanup test execution & output.

- Stop writing everything to files. - Make test output more standard. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@77074 91177308-0d34-0410-b5e6-96231b3b80d8
2009-07-25 14:46:05 +00:00 · 2009-07-25 14:46:05 +00:00 · a957d9996e
--- a/utils/test/MultiTestRunner.py
+++ b/utils/test/MultiTestRunner.py
@ -23,20 +23,14 @@ kTestFileExtensions = set(['.mi','.i','.c','.cpp','.m','.mm','.ll'])

 def getTests(inputs):
    for path in inputs:
-        # Always use absolte paths.
-        path = os.path.abspath(path)
        if not os.path.exists(path):
            print >>sys.stderr,"WARNING: Invalid test \"%s\""%(path,)
            continue
        
-        if os.path.isdir(path):
+        if not os.path.isdir(path):
+            yield path
+
        for dirpath,dirnames,filenames in os.walk(path):
-                dotTests = os.path.join(dirpath,'.tests')
-                if os.path.exists(dotTests):
-                    for ln in open(dotTests):
-                        if ln.strip():
-                            yield os.path.join(dirpath,ln.strip())
-                else:
            # FIXME: This doesn't belong here
            if 'Output' in dirnames:
                dirnames.remove('Output')
@ -44,8 +38,6 @@ def getTests(inputs):
                base,ext = os.path.splitext(f)
                if ext in kTestFileExtensions:
                    yield os.path.join(dirpath,f)
-        else:
-            yield path

 class TestingProgressDisplay:
    def __init__(self, opts, numTests, progressBar=None):
@ -92,22 +84,21 @@ class TestingProgressDisplay:
            else:
                sys.stdout.write('\n')

-        extra = ''
-        if tr.code==TestStatus.Invalid:
-            extra = ' - (Invalid test)'
-        elif tr.failed():
-            extra = ' - %s'%(TestStatus.getName(tr.code).upper(),)
-        print '%*d/%*d - %s%s'%(self.digits, index+1, self.digits, 
-                              self.numTests, tr.path, extra)
+        status = TestStatus.getName(tr.code).upper()
+        print '%s: %s (%*d of %*d)' % (status, tr.path, 
+                                       self.digits, index+1, 
+                                       self.digits, self.numTests)

        if tr.failed() and self.opts.showOutput:
-            TestRunner.cat(tr.testResults, sys.stdout)
+            print "%s TEST '%s' FAILED %s" % ('*'*20, tr.path, '*'*20)
+            print tr.output
+            print "*" * 20

 class TestResult:
-    def __init__(self, path, code, testResults, elapsed):
+    def __init__(self, path, code, output, elapsed):
        self.path = path
        self.code = code
-        self.testResults = testResults
+        self.output = output
        self.elapsed = elapsed

    def failed(self):
@ -154,12 +145,7 @@ class Tester(threading.Thread):
            self.runTest(item)

    def runTest(self, (path, index)):
-        command = path
        base = TestRunner.getTestOutputBase('Output', path)
-        output = base + '.out'
-        testname = path
-        testresults = base + '.testresults'
-        TestRunner.mkdir_p(os.path.dirname(testresults))
        numTests = len(self.provider.tests)
        digits = len(str(numTests))
        code = None
@ -170,9 +156,8 @@ class Tester(threading.Thread):
                code = None
            else:
                startTime = time.time()
-                code = TestRunner.runOneTest(path, command, output, testname, 
-                                             opts.clang, opts.clangcc,
-                                             output=open(testresults,'w'))
+                code, output = TestRunner.runOneTest(path, base, 
+                                                     opts.clang, opts.clangcc)
                elapsed = time.time() - startTime
        except KeyboardInterrupt:
            # This is a sad hack. Unfortunately subprocess goes
@ -180,8 +165,7 @@ class Tester(threading.Thread):
            print 'Ctrl-C detected, goodbye.'
            os.kill(0,9)

-        self.provider.setResult(index, TestResult(path, code, testresults, 
-                                                  elapsed))
+        self.provider.setResult(index, TestResult(path, code, output, elapsed))

 def detectCPUs():
    """
--- a/utils/test/TestRunner.py
+++ b/utils/test/TestRunner.py
@ -57,54 +57,29 @@ def mkdir_p(path):
            if e.errno != errno.EEXIST:
                raise

-def remove(path):
-    try:
-        os.remove(path)
-    except OSError:
-        pass
-
-def cat(path, output):
-    f = open(path)
-    output.writelines(f)
-    f.close()
-
-def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC,
-               output=sys.stdout):
-    OUTPUT = os.path.abspath(OUTPUT)
+import StringIO
+def runOneTest(testPath, tmpBase, clang, clangcc):
+    # Make paths absolute.
+    tmpBase = os.path.abspath(tmpBase)
+    testPath = os.path.abspath(testPath)

    # Create the output directory if it does not already exist.
-    mkdir_p(os.path.dirname(OUTPUT))

-    scriptFile = FILENAME
-            
-    # Verify the script contains a run line.
-    for ln in open(scriptFile):
-        if 'RUN:' in ln:
-            break
-    else:
-        print >>output, "******************** TEST '%s' HAS NO RUN LINE! ********************"%(TESTNAME,)
-        output.flush()
-        return TestStatus.Fail
-
-    FILENAME = os.path.abspath(FILENAME)
-    SCRIPT = OUTPUT + '.script'
+    mkdir_p(os.path.dirname(tmpBase))
+    script = tmpBase + '.script'
    if kSystemName == 'Windows':
-        SCRIPT += '.bat'
-    TEMPOUTPUT = OUTPUT + '.tmp'
+        script += '.bat'

-    substitutions = [('%s',SUBST),
-                     ('%S',os.path.dirname(SUBST)),
-                     ('%llvmgcc','llvm-gcc -emit-llvm -w'),
-                     ('%llvmgxx','llvm-g++ -emit-llvm -w'),
-                     ('%prcontext','prcontext.tcl'),
-                     ('%t',TEMPOUTPUT),
-                     (' clang ', ' ' + CLANG + ' '),
-                     (' clang-cc ', ' ' + CLANGCC + ' ')]
+    substitutions = [('%s', testPath),
+                     ('%S', os.path.dirname(testPath)),
+                     ('%t', tmpBase + '.tmp'),
+                     (' clang ', ' ' + clang + ' '),
+                     (' clang-cc ', ' ' + clangcc + ' ')]

    # Collect the test lines from the script.
    scriptLines = []
    xfailLines = []
-    for ln in open(scriptFile):
+    for ln in open(testPath):
        if 'RUN:' in ln:
            # Isolate the command to run.
            index = ln.index('RUN:')
@ -118,6 +93,10 @@ def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC,
        # FIXME: Support something like END, in case we need to process large
        # files.

+    # Verify the script contains a run line.
+    if not scriptLines:
+        return (TestStatus.Fail, "Test has no run line!")
+    
    # Apply substitutions to the script.
    def processLine(ln):
        # Apply substitutions
@ -133,19 +112,15 @@ def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC,
        ln = scriptLines[i]

        if not ln.endswith('&&'):
-            print >>output, "MISSING \'&&\': %s" % ln
-            print >>output, "FOLLOWED BY   : %s" % scriptLines[i + 1]
-            return TestStatus.Fail
+            return (TestStatus.Fail, 
+                    "MISSING \'&&\': %s\n" +
+                    "FOLLOWED BY   : %s\n" % (ln,scriptLines[i + 1]))
    
        # Strip off '&&'
        scriptLines[i] = ln[:-2]

-    if xfailLines:
-        print >>output, "XFAILED '%s':"%(TESTNAME,)
-        output.writelines(xfailLines)
-
    # Write script file
-    f = open(SCRIPT,'w')
+    f = open(script,'w')
    if kSystemName == 'Windows':
        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(scriptLines))
    else:
@ -153,52 +128,53 @@ def runOneTest(FILENAME, SUBST, OUTPUT, TESTNAME, CLANG, CLANGCC,
    f.write('\n')
    f.close()

-    outputFile = open(OUTPUT,'w')
    p = None
    try:
        if kSystemName == 'Windows':
-            command = ['cmd','/c', SCRIPT]
+            command = ['cmd','/c', script]
        else:
-            command = ['/bin/sh', SCRIPT]
+            command = ['/bin/sh', script]
        
        p = subprocess.Popen(command,
-                             cwd=os.path.dirname(FILENAME),
+                             cwd=os.path.dirname(testPath),
                             stdin=subprocess.PIPE,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             env=kChildEnv)
        out,err = p.communicate()
-        outputFile.write(out)
-        outputFile.write(err)
-        SCRIPT_STATUS = p.wait()
+        exitCode = p.wait()

        # Detect Ctrl-C in subprocess.
-        if SCRIPT_STATUS == -signal.SIGINT:
+        if exitCode == -signal.SIGINT:
            raise KeyboardInterrupt
    except KeyboardInterrupt:
        raise
-    outputFile.close()

    if xfailLines:
-        SCRIPT_STATUS = not SCRIPT_STATUS
-
-    if SCRIPT_STATUS:
-        print >>output, "******************** TEST '%s' FAILED! ********************"%(TESTNAME,)
-        print >>output, "Command: "
-        output.writelines(scriptLines)
-        print >>output, "Incorrect Output:"
-        cat(OUTPUT, output)
-        print >>output, "******************** TEST '%s' FAILED! ********************"%(TESTNAME,)
-        output.flush()
-        if xfailLines:
-            return TestStatus.XPass
+        ok = exitCode != 0
+        status = (TestStatus.XPass, TestStatus.XFail)[ok]
    else:
-            return TestStatus.Fail
+        ok = exitCode == 0
+        status = (TestStatus.Fail, TestStatus.Pass)[ok]

-    if xfailLines:
-        return TestStatus.XFail
-    else:
-        return TestStatus.Pass
+    if ok:
+        return (status,'')
+
+    output = StringIO.StringIO()
+    print >>output, "Script:"
+    print >>output, "--"
+    print >>output, '\n'.join(scriptLines)
+    print >>output, "--"
+    print >>output, "Exit Code: %r" % exitCode
+    print >>output, "Command Output (stdout):"
+    print >>output, "--"
+    output.write(out)
+    print >>output, "--"
+    print >>output, "Command Output (stderr):"
+    print >>output, "--"
+    output.write(err)
+    print >>output, "--"
+    return (status, output.getvalue())

 def capture(args):
    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -304,14 +280,17 @@ def main():
        opts.clangcc = inferClangCC(opts.clang)

    for path in args:
-        command = path
-        output = getTestOutputBase('Output', path) + '.out'
-        testname = path
+        base = getTestOutputBase('Output', path) + '.out'
        
-        res = runOneTest(path, command, output, testname, 
-                         opts.clang, opts.clangcc)
+        status,output = runOneTest(path, base, opts.clang, opts.clangcc)
+        print '%s: %s' % (TestStatus.getName(status).upper(), path)
+        if status == TestStatus.Fail or status == TestStatus.XPass:
+            print "%s TEST '%s' FAILED %s" % ('*'*20, path, '*'*20)
+            sys.stdout.write(output)
+            print "*" * 20
+            sys.exit(1)

-    sys.exit(res == TestStatus.Fail or res == TestStatus.XPass)
+    sys.exit(0)

 if __name__=='__main__':
    main()