Merge pull request #452 from libengu/CheckPass

Add llilc_checkpass Python Script.

test/applyfilter.py

@@ -3,6 +3,9 @@
 #title           :applyfilter.py
 #description     :
 # This script performs normalization of verbose LLVM IR dump generated by LLILC JIT.
+# The module also provides support facilty for llilc_runtest and llilc_checkpass
+# such as ApplyAll etc.
+# 
 # Normalized cases include:
 # 
 # Suppress address difference from run to run
@@ -52,37 +55,67 @@ import sys
 import re
 import argparse
 
-# Apply filter on src and create a normalized file dest
-def ApplyOne(src, dest):
+# Apply filter on src and create a normalized file dest.
+# And extract out function summary if specified.
+def ApplyOne(src, dest, summary=None):
     re_addr = re.compile(r'i64 \d{10}\d*')
     re_type = re.compile(r'%("?)(.*?)\.\d+\1 addrspace')
     re_phi = re.compile(r'\[ \d{10}\d*, %')
+    if summary is None:
         with open(src, 'r') as ins, open(dest, 'w') as outs:
             for line in ins:
                 line = re_addr.sub(r'i64 NORMALIZED_ADDRESS', line)
                 line = re_type.sub(r'%\1\2.NORMALIZED_TYPEID\1 addrspace', line)
                 line = re_phi.sub(r'[ NORMALIZED_ADDRESS, %', line)
                 outs.write(line)
+    else:
+        re_read_failure = re.compile(r'Failed to read ')
+        re_read_success = re.compile(r'Successfully read ')
+        with open(src, 'r') as ins, open(dest, 'w') as outs, open(summary, 'w') as sums:
+            for line in ins:
+                extract = re_read_failure.search(line)
+                if extract is None:
+                    extract = re_read_success.search(line)
+                if extract is not None:
+                    sums.write(line)
+                else:
+                    line = re_addr.sub(r'i64 NORMALIZED_ADDRESS', line)
+                    line = re_type.sub(r'%\1\2.NORMALIZED_TYPEID\1 addrspace', line)
+                    line = re_phi.sub(r'[ NORMALIZED_ADDRESS, %', line)
+                outs.write(line)
 
-# Apply filter recursively on directory walk_dir
+# Apply filter recursively on directory walk_dir.
+# And extract out function summary.
 def ApplyAll(walk_dir):
-    for root, subdirs, files in os.walk(walk_dir):
-        for filename in files:
-            if filename.endswith("error.txt"):
-                tmp_filename = filename + ".tmp"
-                file_path = os.path.join(root, filename)
-                tmp_file_path = os.path.join(root, tmp_filename)
-                print("Normalizing " + file_path)
-                ApplyOne(file_path, tmp_file_path)
+    for root, sub_dirs, files in os.walk(walk_dir):
+        for file_name in files:
+            if file_name.endswith('error.txt'):
+                sum_file_name = str(file_name).replace('error.txt', 'sum.txt')
+                tmp_file_name = file_name + '.tmp'
+                file_path = os.path.join(root, file_name)
+                sum_file_path = os.path.join(root, sum_file_name)
+                tmp_file_path = os.path.join(root, tmp_file_name)
+                ApplyOne(file_path, tmp_file_path, sum_file_path)
                 os.remove(file_path)
                 os.rename(tmp_file_path, file_path)
 
+# Rename file name in summary result to match what ApplyAll may create
+# so that llilc_checkpass can perform checking.
+def SummaryRenameAll(walk_dir):
+    for root, sub_dirs, files in os.walk(walk_dir):
+        for file_name in files:
+            if file_name.endswith('error.txt'):
+                sum_file_name = str(file_name).replace('error.txt', 'sum.txt')
+                file_path = os.path.join(root, file_name)
+                sum_file_path = os.path.join(root, sum_file_name)
+                os.rename(file_path, sum_file_path)
+
 # The script itself applies the filter on one file
 if __name__=='__main__':
     # Parse the command line
     parser = argparse.ArgumentParser()
-    parser.add_argument("src", type=str, help="source result to apply filter on")
-    parser.add_argument("dest", type=str, help="destination result after applying filter")
+    parser.add_argument('src', type=str, help='source result to apply filter on')
+    parser.add_argument('dest', type=str, help='destination result after applying filter')
     args = parser.parse_args()
 
     # Apply the filter on one file

test/const.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+#
+#title           :const.py
+#description     :
+#
+# const class does not allow rebinding value to name so that name is constant.
+#
+#==========================================================================================
+
+class _const:
+    class ConstError(TypeError): pass
+    def __setattr__(self,name,value):
+        if name in self.__dict__:
+            message = "Can't rebind const " + name
+            raise self.ConstError(message)
+        self.__dict__[name]=value
+import sys
+sys.modules[__name__]=_const()

test/llilc_checkpass.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+#
+#title           :llilc_checkpass.py
+#description     :
+# 
+# llilc_checkpass compares two test results, no matter summary or verbose,
+# and checks if any function successfully jitted by LLILC in base result
+# failed jitting in diff result.
+#
+# If the check passed, the script return 0. In case of 0, it might have
+# jitted more functions in diff result than in base result. The two results
+# are not necessary the same in this case. Counting result is reported.
+# 
+# If the check failed due to newly faily routine, return 1.
+#     
+# If the check failed due to unexpected reason, return negative numbers.
+#
+# usage: llilc_checkpass.py [-h] -b BASE_RESULT_PATH -d DIFF_RESULT_PATH
+# 
+# optional arguments:
+#   -h, --help            show this help message and exit
+# 
+# required arguments:
+#   -b BASE_RESULT_PATH, --base-result-path BASE_RESULT_PATH
+#                         full path to base result
+#   -d DIFF_RESULT_PATH, --diff-result-path DIFF_RESULT_PATH
+#                         full path to diff result
+#
+#==========================================================================================
+
+import argparse
+import re
+import sys
+import os
+import difflib
+import const
+
+def main(argv):
+    # define return code const value
+    const.CheckPassFail = 1
+    const.CheckPassOK = 0
+    const.GeneralError = -1
+    const.UnknownArguments = -2
+    const.MissingResult = -3
+
+    # Parse the command line    
+    parser = argparse.ArgumentParser()
+    required = parser.add_argument_group('required arguments')
+    required.add_argument('-b', '--base-result-path', type=str, required=True,
+                        help='full path to base result')
+    required.add_argument('-d', '--diff-result-path', type=str, required=True,
+                        help='full path to diff result')
+    args, unknown = parser.parse_known_args(argv)
+
+    if unknown:
+        print('Unknown argument(s): ', ', '.join(unknown))
+        return const.UnknownArguments
+
+    try:
+        # Collect a list of summary result file relative path in base result    
+        base_files = []
+        for root, dirs, files in os.walk(args.base_result_path):
+            for file in files:
+                if file.endswith('sum.txt'):
+                    relative_path = os.path.relpath(root, args.base_result_path)
+                    relative_path = os.path.join(relative_path, file)
+                    base_files.append(relative_path)
+ 
+        # Collect a list of summary result file relative path in diff result    
+        diff_files = []
+        for root, dirs, files in os.walk(args.diff_result_path):
+            for file in files:
+                if file.endswith('sum.txt'):
+                    relative_path = os.path.relpath(root, args.diff_result_path)
+                    relative_path = os.path.join(relative_path, file)
+                    diff_files.append(relative_path)
+    except:
+        e = sys.exc_info()[0]
+        print('Error: CheckPass failed due to ', e)
+        return const.GeneralError
+
+    # Check if results are empty
+    if len(base_files) == 0:
+        print('Error: base result is empty')
+        return const.MissingResult
+
+    if len(diff_files) == 0:
+        print('Error: diff result is empty')
+        return const.MissingResult
+
+    # Counting the newly failed or passed test cases
+    count_new_failed = 0
+    count_new_passed = 0
+
+    print('Checking started.')    
+    for file in base_files:
+        if file in diff_files:
+            try:
+                base_file_path = os.path.join(args.base_result_path, file)
+                diff_file_path = os.path.join(args.diff_result_path, file)
+                with open(base_file_path, 'r') as bases, open(diff_file_path, 'r') as diffs:
+                    diff = difflib.ndiff(bases.readlines(),diffs.readlines())
+                    for line in diff:
+                        if re.search('- Successfully read ', line):
+                            count_new_failed = count_new_failed + 1
+                        if re.search('\+ Successfully read ', line):
+                           count_new_passed = count_new_passed + 1
+            except:
+                e = sys.exc_info()[0]
+                print('Error: CheckPass failed due to ', e)
+                return const.GeneralError
+        else:
+            missing_result_file = os.path.join(args.diff_result_path, file)
+            print('Error: diff result does not result file ', missing_result_file)
+            return const.MissingResult
+
+    print('CheckPass: ', count_new_failed, ' test cases passed in base result but failed in diff result.')
+    print('CheckPass: ', count_new_passed, ' test cases failed in base result but passed in diff result.') 
+    if count_new_failed == 0:
+        print('CheckPass Passed.')
+        return const.CheckPassOK
+    else:
+        print('CheckPass Failed.\n')
+        return const.CheckPassFail
+
+if __name__ == '__main__':
+    return_code = main(sys.argv[1:])
+    sys.exit(return_code)

test/llilc_runtest.py

@@ -14,13 +14,13 @@
 #
 # To exclude undesired test cases, please edit exclusion file.
 #
-# usage: llilc_runtest.py [-h] [-a {x86,x64}] [-b {debug,release}]
+# usage: llilc_runtest.py [-h] [-a {x64,x86}] [-b {debug,release}]
 #                         [-d {summary,verbose}] [-r RESULT_PATH] -j JIT_PATH -c
 #                         CORECLR_RUNTIME_PATH
 # 
 # optional arguments:
 #   -h, --help            show this help message and exit
-#   -a {x86,x64}, --arch {x86,x64}
+#   -a {x64,x86}, --arch {x64,x86}
 #                         the target architure
 #   -b {debug,release}, --build {debug,release}
 #                         release or debug build of CoreCLR run-time used
@@ -28,6 +28,8 @@
 #                         the dump level: summary, or verbose
 #   -r RESULT_PATH, --result-path RESULT_PATH
 #                         the path to runtest result output directory
+# 
+# required arguments:
 #   -j JIT_PATH, --jit-path JIT_PATH
 #                         full path to jit .dll
 #   -c CORECLR_RUNTIME_PATH, --coreclr-runtime-path CORECLR_RUNTIME_PATH
@@ -41,38 +43,40 @@ import os
 import shutil
 import glob
 import sys
+import stat
 import subprocess
 import applyfilter
+import const
 
 # Return OS name used internally in paths
 def OSName():
-    if os.name == "nt":
-        return "Windows_NT"
+    if os.name == 'nt':
+        return 'Windows_NT'
     else:
-        return "Unknown"
+        return 'Unknown'
 
 # Return the path of built test path relavtive to coreclr tests directory
 def BuiltTestPath(arch, build):
-    built_test_directory = OSName() + "." + arch + "." + build
-    built_test_path = os.path.join("..", "bin", "tests", built_test_directory)
+    built_test_directory = OSName() + '.' + arch + '.' + build
+    built_test_path = os.path.join('..', 'bin', 'tests', built_test_directory)
     return built_test_path
 
 # Exculde top level test directories
 def ExcludeTopLevelTestDirectories():
-    exclusion = os.path.join(os.path.dirname(__file__), "exclusion")
+    exclusion = os.path.join(os.path.dirname(__file__), 'exclusion')
     with open(str(exclusion)) as excl:
         content = excl.readlines()
-        os.environ["SkipTestAssemblies"] = content[1]
+        os.environ['SkipTestAssemblies'] = content[1]
 
 # Exclude individual test cases
 def ExcludeIndividualTestCases(path):
-    exclusion = os.path.join(os.path.dirname(__file__), "exclusion")
+    exclusion = os.path.join(os.path.dirname(__file__), 'exclusion')
     with open(str(exclusion)) as excl:
         try:
             glob_files_to_delete = []
             content = excl.read().splitlines()
             for line in range(4, len(content)):
-                all = content[line] + "*"
+                all = content[line] + '*'
                 files_to_delete = os.path.join(path, all) 
                 glob_files_to_delete.extend(glob.glob(files_to_delete))
             for file_to_delete in glob_files_to_delete:
@@ -85,7 +89,7 @@ def CleanUpResultFiles(path):
     try:
         for root, subdirs, files in os.walk(path):
             for file in files:
-                if not file.endswith("error.txt"):
+                if not file.endswith('error.txt'):
                     file_path = os.path.join(root, file)
                     os.remove(file_path)
     except OSError:
@@ -93,8 +97,8 @@ def CleanUpResultFiles(path):
 
 # Return default result path if result is generated
 def DefaultResultPath():
-    default_result_path = os.environ["TEMP"]
-    default_result_path = os.path.join(default_result_path, "LLILCTestResult")
+    default_result_path = os.environ['TEMP']
+    default_result_path = os.path.join(default_result_path, 'LLILCTestResult')
     return default_result_path
 
 # Remove readonly files
@@ -102,92 +106,140 @@ def del_rw(action, name, exc):
     os.chmod(name, stat.S_IWRITE)
     os.remove(name)
 
+# Count files in a directory
+def CountFiles(path, suffix):
+    total = 0
+    for root, sub_dirs, files in os.walk(path):
+        for file_name in files:
+            if file_name.endswith(suffix):
+                total = total + 1
+    return total
+
 def main(argv):
+    # define return code const value
+    const.RunTestFail = 1
+    const.RunTestOK = 0
+    const.GeneralError = -1
+    const.UnknownArguments = -2
+    const.NormalizationFail = -3
+    const.WrongRunDir = -4
+
     # Parse the command line    
     parser = argparse.ArgumentParser()
-    parser.add_argument("-a", "--arch", type=str, choices={"x86", "x64"},
-                        default="x64", help="the target architure")
-    parser.add_argument("-b", "--build", type=str, choices={"release", "debug"}, 
-                        default="debug", help="release or debug build of CoreCLR run-time used")
-    parser.add_argument("-d", "--dump-level", type=str, choices={"summary", "verbose"}, 
-                        help="the dump level: summary, or verbose")
-    parser.add_argument("-r", "--result-path", type=str, 
-                        default=DefaultResultPath(), help="the path to runtest result output directory")
-    parser.add_argument("-j", "--jit-path", required=True, 
-                        help="full path to jit .dll")
-    parser.add_argument("-c", "--coreclr-runtime-path", required=True, 
-                        help="full path to CoreCLR run-time binary directory")
+    parser.add_argument('-a', '--arch', type=str, choices={'x86', 'x64'},
+                        default='x64', help='the target architure')
+    parser.add_argument('-b', '--build', type=str, choices={'release', 'debug'}, 
+                        default='debug', help='release or debug build of CoreCLR run-time used')
+    parser.add_argument('-d', '--dump-level', type=str, choices={'summary', 'verbose'}, 
+                        help='the dump level: summary, or verbose')
+    parser.add_argument('-r', '--result-path', type=str, 
+                        default=DefaultResultPath(), help='the path to runtest result output directory')
+    required = parser.add_argument_group('required arguments')
+    required.add_argument('-j', '--jit-path', required=True, 
+                        help='full path to jit .dll')
+    required.add_argument('-c', '--coreclr-runtime-path', required=True, 
+                        help='full path to CoreCLR run-time binary directory')
     args, unknown = parser.parse_known_args(argv)
 
     if unknown:
-        print("Unknown argument(s): ", ", ".join(unknown))
-        return -2
+        print('Unknown argument(s): ', ', '.join(unknown))
+        return const.UnknowArguments
         
     # Ensure the command run from a CoreCLR tests directory
     current_work_directory = os.getcwd()
     path, folder = os.path.split(current_work_directory)
     parent_path, parent_folder = os.path.split(path)
-    if (not folder == "tests" or
-        not parent_folder == "coreclr"):
-        print("This script is required to run from tests directory in a CoreCLR repository." )    
-        return -1
+    if (not folder == 'tests' or
+        not parent_folder == 'coreclr'):
+        print('This script is required to run from tests directory in a CoreCLR repository.' )    
+        return const.WrongRunDir
 
+    try:
         # Determine the built test location
         build_test_path = BuiltTestPath(str(args.arch), str(args.build))
 
         # Determine time stamp
-    time_stamp = str(time.time()).split(".")[0]
+        time_stamp = str(time.time()).split('.')[0]
 
         # Copy in llilcjit.dll with time stamp
-    time_stamped_jit_name = "LLILCJit" + time_stamp + ".dll"
+        time_stamped_jit_name = 'LLILCJit' + time_stamp + '.dll'
         time_stamped_jit_path = os.path.join(args.coreclr_runtime_path, time_stamped_jit_name)
         shutil.copy2(args.jit_path, time_stamped_jit_path)
 
         # Create llilctestenv.cmd with time stamp
-    time_stamped_test_env_name = "LLILCTestEnv" + time_stamp + ".cmd"
+        time_stamped_test_env_name = 'LLILCTestEnv' + time_stamp + '.cmd'
         time_stamped_test_env_path = os.path.join(args.coreclr_runtime_path, time_stamped_test_env_name)
 
         # Todo: Test Env is right now only for Windows. Will expand when cross platform
-    with open(time_stamped_test_env_path, "w") as test_env:
-        test_env.write("set COMPlus_AltJit=*\n")
-        test_env.write("set COMPlus_AltJitName=" + time_stamped_jit_name + "\n")
-        test_env.write("set COMPlus_GCConservative=1\n")
-        test_env.write("chcp 65001\n")
+        with open(time_stamped_test_env_path, 'w') as test_env:
+            test_env.write('set COMPlus_AltJit=*\n')
+            test_env.write('set COMPlus_AltJitName=' + time_stamped_jit_name + '\n')
+            test_env.write('set COMPlus_GCConservative=1\n')
+            test_env.write('chcp 65001\n')
             if args.dump_level is not None:
-            test_env.write("set COMPlus_DumpLLVMIR=" + args.dump_level + "\n")
+                test_env.write('set COMPlus_DumpLLVMIR=' + args.dump_level + '\n')
 
         # Exclude undesired tests from running
         ExcludeTopLevelTestDirectories()
         ExcludeIndividualTestCases(build_test_path)
+    except:
+        e = sys.exc_info()[0]
+        print('Error: RunTest failed due to ', e)
+        return const.GeneralError
 
     # Run the test
-    return_code = 0
-    runtest_command = "runtest " + args.arch + " " + args.build 
-    runtest_command = runtest_command + " Testenv " + time_stamped_test_env_path
-    runtest_command = runtest_command + " " + args.coreclr_runtime_path 
+    return_code = const.RunTestOK
+    runtest_command = 'runtest ' + args.arch + ' ' + args.build 
+    runtest_command = runtest_command + ' Testenv ' + time_stamped_test_env_path
+    runtest_command = runtest_command + ' ' + args.coreclr_runtime_path 
     print(runtest_command)
     error_level = subprocess.call(runtest_command, shell=True)
     if error_level == 1:
-        return_code = 1
+        return_code = const.RunTestFail
 
     # Remove temporary time-stamped jit and test env files
+    try:
         os.remove(time_stamped_jit_path)
         os.remove(time_stamped_test_env_path)
+    except:
+        e = sys.exc_info()[0]
+        print('Error: RunTest failed due to ', e)
+        return const.GeneralError
 
     # Copy out result if there is one, clean up undesired files, 
-    # normalize it in case of verbose dump.
+    # In case of verbose result, normalize it and extract out summary.
+    # In case of summary result, rename all result file name.
     if args.dump_level is not None:
-        coreclr_result_path = os.path.join(build_test_path, "Reports")
+        try:
+            coreclr_result_path = os.path.join(build_test_path, 'Reports')
+            if os.path.exists(args.result_path):
                 shutil.rmtree(args.result_path, onerror=del_rw)            
             shutil.copytree(coreclr_result_path, args.result_path)
             ExcludeIndividualTestCases(args.result_path)
             CleanUpResultFiles(args.result_path)
-        if args.dump_level == "verbose":
-            print("Applying filter...")
+            total = CountFiles(args.result_path, 'error.txt')
+            if args.dump_level == 'verbose':
+                print('Verbose-mode post-processing started')
+                print('found ', total, 'valid raw test outputs (error.txt) under ', str(coreclr_result_path))
+                print('creating normalized outputs (error.txt) under ', str(args.result_path))
+                print('creating summary outputs (sum.txt) under ', str(args.result_path))
                 applyfilter.ApplyAll(args.result_path)
+                print('Verbose-mode post-processing finished')
+            if args.dump_level == 'summary':
+                print('Summary-mode post-processing started')
+                print('found ', total, 'valid raw test outputs (error.txt) under ', os.path.abspath(coreclr_result_path))
+                print('creating summary outputs (sum.txt) under ', str(args.result_path))
+                applyfilter.SummaryRenameAll(args.result_path) 
+                print('Summary-mode post-processing finished')
+        except:
+            e = sys.exc_info()[0]
+            print('Error: Test result normalization failed due to ', e)
+            return const.NormalizationFail
+    else:
+        print('No post-processing needed.')
 
     return return_code
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     return_code = main(sys.argv[1:])
     sys.exit(return_code)