Quality of Life improvements to CoreFX testing infra (#5651)

* QoL Improvements

* Test Cleanup

* Add OS detection to non-Windows tests

tests/CoreFX/build-and-run-test.cmd

@@ -51,4 +51,4 @@ if not exist "%TestFolder%\native\%TestExecutable%".exe (
     exit /b 1
 )
 
-call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %XunitLogDir%\%TestFileName%.xml 
+call %TestFolder%\native\%TestExecutable% %TestFolder%\%TestFileName%.dll -xml %XunitLogDir%\%TestFileName%.xml -notrait category=nonnetcoreapptests -notrait category=nonwindowstests  -notrait category=failing

tests/CoreFX/corerun

@@ -51,7 +51,23 @@ fi
 
 echo Executing ${TestFileName} - writing logs to ${LogDir}/${TestFileName}.xml
 chmod +x ${TestFolderName}/publish/${TestExecutable} 
-${TestFolderName}/publish/${TestExecutable} ${TestFolderName}/${TestFileName}.dll -xml ${LogDir}/${TestFileName}.xml
+
+case "$(uname -s)" in 
+    # Check if we're running under Linux
+    Linux)
+    export OSCategory="nonlinuxtests"
+    ;;
+    # Check if we're running under OSX
+    Darwin)
+    export OSCategory="nonosxtests"
+    ;;
+    # Default to Linux if we don't recognize the OS
+    *)
+    export OSCategory="nonlinuxtests"
+    ;;
+esac
+
+${TestFolderName}/publish/${TestExecutable} ${TestFolderName}/${TestFileName}.dll -xml ${LogDir}/${TestFileName}.xml -notrait category=nonnetcoreapptests -notrait category=${OSCategory}  -notrait category=failing
 export __exitcode=$?
 
 exit ${__exitcode}

tests/CoreFX/runtest/runtest.sh

@@ -69,3 +69,5 @@ if [ -z "$__LogDir" ]; then
 fi
 
 run_tests_in_directory ${testRootDir}
+
+exit $EXIT_CODE_SUCCESS

tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs

@@ -45,52 +45,58 @@ namespace CoreFX.TestUtils.TestFileSetup
             ExitCode exitCode = ExitCode.UnknownError;
             ArgumentSyntax argSyntax = ParseCommandLine(args);
 
-            if (!Directory.Exists(outputDir))
-            {
-                try
-                {
-                    Directory.CreateDirectory(outputDir);
-                }
-                catch (IOException)
-                {
-                    exitCode = ExitCode.IOError;
-                    Environment.Exit((int)exitCode);
-                }
-            }
-
-            // parse args
             try
             {
+                if (!Directory.Exists(outputDir))
+                    Directory.CreateDirectory(outputDir);
+
+                if (cleanTestBuild)
+                {
+                    CleanBuild(outputDir);
+                }
+
                 SetupTests(testUrl, outputDir, ReadTestNames(testListPath)).Wait();
                 exitCode = ExitCode.Success;
+            }
 
-            }
-            catch (HttpRequestException)
+            catch (AggregateException e)
             {
-                exitCode = ExitCode.HttpError;
-            }
-            catch (IOException)
-            {
-                exitCode = ExitCode.IOError;
+                e.Handle(innerExc =>
+                {
+                    if (innerExc is HttpRequestException)
+                    {
+                        exitCode = ExitCode.HttpError;
+                        Console.WriteLine("Error downloading tests from: " + testUrl);
+                        Console.WriteLine(innerExc.Message);
+                        return true;
+                    }
+                    else if (innerExc is IOException)
+                    {
+                        exitCode = ExitCode.IOError;
+                        Console.WriteLine(innerExc.Message);
+                        return true;
+                    }
+                    return false;
+                });
             }
 
             Environment.Exit((int)exitCode);
         }
 
-        public static ArgumentSyntax ParseCommandLine(string[] args)
+        private static ArgumentSyntax ParseCommandLine(string[] args)
         {
             ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax =>
             {
                 syntax.DefineOption("out|outDir|outputDirectory", ref outputDir, "Directory where tests are downloaded");
                 syntax.DefineOption("testUrl", ref testUrl, "URL, pointing to the list of tests");
                 syntax.DefineOption("testListJsonPath", ref testListPath, "JSON-formatted list of test assembly names to download");
-                syntax.DefineOption("clean", ref cleanTestBuild, "Remove all previously built test assemblies");
+                syntax.DefineOption("clean|cleanOutputDir", ref cleanTestBuild, "Clean test assembly output directory");
             });
 
             return argSyntax;
         }
 
-        public static Dictionary<string, string> ReadTestNames(string testFilePath)
+        private static Dictionary<string, string> ReadTestNames(string testFilePath)
         {
             Debug.Assert(File.Exists(testFilePath));
 
@@ -112,7 +118,7 @@ namespace CoreFX.TestUtils.TestFileSetup
             return testNames;
         }
 
-        public static async Task SetupTests(string jsonUrl, string destinationDirectory, Dictionary<string, string> testNames = null, bool runAllTests = false)
+        private static async Task SetupTests(string jsonUrl, string destinationDirectory, Dictionary<string, string> testNames = null, bool runAllTests = false)
         {
             Debug.Assert(Directory.Exists(destinationDirectory));
             Debug.Assert(runAllTests || testNames != null);
@@ -124,12 +130,18 @@ namespace CoreFX.TestUtils.TestFileSetup
             }
             Dictionary<string, string> testPayloads = await GetTestUrls(jsonUrl, testNames, runAllTests);
 
+            if (testPayloads == null)
+            {
+                return;
+            }
+
             await GetTestArchives(testPayloads, tempDirPath);
             ExpandArchivesInDirectory(tempDirPath, destinationDirectory);
 
             Directory.Delete(tempDirPath);
         }
-        public static async Task<Dictionary<string, string>> GetTestUrls(string jsonUrl, Dictionary<string, string> testNames = null, bool runAllTests = false)
+
+        private static async Task<Dictionary<string, string>> GetTestUrls(string jsonUrl, Dictionary<string, string> testNames = null, bool runAllTests = false)
         {
             if (httpClient is null)
             {
@@ -137,7 +149,6 @@ namespace CoreFX.TestUtils.TestFileSetup
             }
 
             Debug.Assert(runAllTests || testNames != null);
-
             // Set up the json stream reader
             using (var responseStream = await httpClient.GetStreamAsync(jsonUrl))
             using (var streamReader = new StreamReader(responseStream))
@@ -180,7 +191,7 @@ namespace CoreFX.TestUtils.TestFileSetup
             return testNames;
         }
 
-        public static async Task GetTestArchives(Dictionary<string, string> testPayloads, string downloadDir)
+        private static async Task GetTestArchives(Dictionary<string, string> testPayloads, string downloadDir)
         {
             if (httpClient is null)
             {
@@ -218,7 +229,7 @@ namespace CoreFX.TestUtils.TestFileSetup
             }
         }
 
-        public static void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true)
+        private static void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true)
         {
             Debug.Assert(Directory.Exists(archiveDirectory));
             Debug.Assert(Directory.Exists(destinationDirectory));
@@ -229,12 +240,6 @@ namespace CoreFX.TestUtils.TestFileSetup
             {
                 string destinationDirName = Path.Combine(destinationDirectory, Path.GetFileNameWithoutExtension(archivePath));
 
-                // If doing clean test build - delete existing artefacts
-                if (Directory.Exists(destinationDirName) && cleanTestBuild)
-                {
-                    Directory.Delete(destinationDirName, true);
-                }
-
                 ZipFile.ExtractToDirectory(archivePath, destinationDirName);
 
 
@@ -246,6 +251,21 @@ namespace CoreFX.TestUtils.TestFileSetup
             }
         }
 
+        private static void CleanBuild(string directoryToClean)
+        {
+            Debug.Assert(Directory.Exists(directoryToClean));
+            DirectoryInfo dirInfo = new DirectoryInfo(directoryToClean);
+
+            foreach (FileInfo file in dirInfo.EnumerateFiles())
+            {
+                file.Delete();
+            }
+
+            foreach (DirectoryInfo dir in dirInfo.EnumerateDirectories())
+            {
+                dir.Delete(true);
+            }
+        }
 
     }
 }

tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs

@@ -38,55 +38,72 @@ namespace CoreFX.TestUtils.XUnit
             int failed = 0;
             int skipped = 0;
             ulong timeElapsed = 0;
+            bool printTotals = false;
 
-            foreach(string logFile in logFiles)
+            foreach (string logFile in logFiles)
             {
-                // XMLReader escapes the character sequence \\.. as just a single backslash \ - Is this intended behavior? 
-                using (XmlReader reader = XmlReader.Create(logFile.Replace(@"\\..", @"\..")))
+                try
                 {
-                    reader.MoveToContent();
-                    reader.ReadToDescendant("collection");
-                    do
+
+                    // XMLReader escapes the character sequence \\.. as just a single backslash \ - Is this intended behavior? 
+                    using (XmlReader reader = XmlReader.Create(logFile.Replace(@"\\..", @"\..")))
                     {
-                        // Get total tests in current element
-                        string totalAttr = reader.GetAttribute("total");
-                        int currentTotal;
-                        Int32.TryParse(totalAttr, out currentTotal);
-                        total += currentTotal;
+                        reader.MoveToContent();
+                        reader.ReadToDescendant("collection");
+                        do
+                        {
+                            // Get total tests in current element
+                            string totalAttr = reader.GetAttribute("total");
+                            int currentTotal;
+                            Int32.TryParse(totalAttr, out currentTotal);
 
-                        // Get passed tests 
-                        string passedAttr = reader.GetAttribute("passed");
-                        int currentPassed;
-                        Int32.TryParse(passedAttr, out currentPassed);
-                        passed += currentPassed;
+                            // Get passed tests 
+                            string passedAttr = reader.GetAttribute("passed");
+                            int currentPassed;
+                            Int32.TryParse(passedAttr, out currentPassed);
 
-                        // Get failed tests
-                        string failedAttr = reader.GetAttribute("failed");
-                        int currentFailed;
-                        Int32.TryParse(failedAttr, out currentFailed);
-                        failed += currentFailed;
+                            // Get failed tests
+                            string failedAttr = reader.GetAttribute("failed");
+                            int currentFailed;
+                            Int32.TryParse(failedAttr, out currentFailed);
 
-                        // Get skipped tests
-                        string skippedAttr = reader.GetAttribute("skipped");
-                        int currentSkipped;
-                        Int32.TryParse(skippedAttr, out currentSkipped);
-                        skipped += currentSkipped;
+                            // Get skipped tests
+                            string skippedAttr = reader.GetAttribute("skipped");
+                            int currentSkipped;
+                            Int32.TryParse(skippedAttr, out currentSkipped);
 
-                        // Get time elapsed
-                        string timeAttr = reader.GetAttribute("time");
-                        ulong currentTime;
-                        UInt64.TryParse(timeAttr, out currentTime);
-                        timeElapsed += currentTime;
+                            // Get time elapsed
+                            string timeAttr = reader.GetAttribute("time");
+                            ulong currentTime;
+                            UInt64.TryParse(timeAttr, out currentTime);
 
-                    } while (reader.ReadToNextSibling("collection"));
+                            // Update running total only once current element has been parsed
+                            total += currentTotal;
+                            passed += currentPassed;
+                            failed += currentFailed;
+                            skipped += currentSkipped;
+                            timeElapsed += currentTime;
 
+                        } while (reader.ReadToNextSibling("collection"));
+
+                        // If we've fully parsed a log, print totals
+                        printTotals = true;
+                    }
+                }
+                catch (XmlException exc)
+                {
+                    Console.WriteLine("Malformed Log: {0} ", logFile);
+                    Console.WriteLine("Reason: {0} ", exc.Message);
+                    continue;
                 }
             }
 
-            Console.WriteLine("=== CoreFX TEST EXECUTION SUMMARY ===: ");
-            Console.WriteLine(String.Format("Total: {0}, Passed: {1}, Failed: {2}, Skipped: {3}", total, passed, failed, timeElapsed));
-            Console.WriteLine("Detailed logs written to: " + logDir);
-
+            if (printTotals)
+            {
+                Console.WriteLine("=== CoreFX TEST EXECUTION SUMMARY ===: ");
+                Console.WriteLine(String.Format("Total: {0}, Passed: {1}, Failed: {2}, Skipped: {3}", total, passed, failed, timeElapsed));
+                Console.WriteLine("Detailed logs written to: " + logDir);
+            }
         }
 
         private static ArgumentSyntax ParseCommandLine(string[] args)
@@ -106,7 +123,7 @@ namespace CoreFX.TestUtils.XUnit
             Debug.Assert(Directory.Exists(logDirectory));
             Console.WriteLine(logDirectory);
             var logFiles = Directory.EnumerateFiles(logDirectory, logPattern, SearchOption.AllDirectories);
-            
+
             return logFiles;
         }
 

tests/runtest.cmd

@@ -383,7 +383,9 @@ goto :eof
     )
 
     "%CoreRT_CliDir%\dotnet.exe" !CoreRT_TestingUtilitiesOutputDir!\%CoreRT_TestFileHelperName%.dll --clean --outputDirectory !CoreRT_TestExtRepo_CoreFX! --testListJsonPath "%TEST_LIST%" --testUrl "%TESTS_REMOTE_URL%"
-
+    if errorlevel 1 (
+        exit /b 1
+    )
 
     exit /b 0
 
@@ -474,7 +476,7 @@ goto :eof
 :TestExtRepoCoreFX
     
     set CoreRT_TestExtRepo_CoreFX=%CoreRT_TestRoot%\..\tests_downloaded\CoreFX
-    set CoreRT_TestingUtilitiesOutputDir=%CoreRT_TestExtRepo_CoreFX%\Utilities
+    set CoreRT_TestingUtilitiesOutputDir=%CoreRT_TestExtRepo_CoreFX%\..\CoreFXUtilities
 
     :: Set paths to helpers
     set CoreRT_TestFileHelperName=CoreFX.TestUtils.TestFileSetup
@@ -492,6 +494,10 @@ goto :eof
     set FXCustomTestLauncher=%CoreRT_TestRoot%\CoreFX\build-and-run-test.cmd
     set XunitTestBinBase=%CoreRT_TestExtRepo_CoreFX%
     set XunitLogDir= %__LogDir%\CoreFX
+    
+    :: Clean up existing logs
+    if exist "%XunitLogDir%" rmdir /S /Q "%XunitLogDir%"
+    mkdir "%XunitLogDir%"
     pushd %CoreRT_TestRoot%\CoreFX\runtest
 
     :: TODO Add single test/target test support; add exclude tests argument

tests/runtest.sh

@@ -206,7 +206,7 @@ run_coreclr_tests()
 run_corefx_tests()
 {
     CoreRT_TestExtRepo_CoreFX=${CoreRT_TestRoot}/../tests_downloaded/CoreFX
-    CoreRT_TestingUtilitiesOutputDir=${CoreRT_TestExtRepo_CoreFX}/Utilities
+    CoreRT_TestingUtilitiesOutputDir=${CoreRT_TestExtRepo_CoreFX}/../CoreFXUtilities
 
     export CoreRT_TestRoot
     export CoreRT_EnableCoreDumps
@@ -235,9 +235,12 @@ run_corefx_tests()
     FXCustomTestLauncher=${CoreRT_TestRoot}/CoreFX/corerun
     XunitTestBinBase=${CoreRT_TestExtRepo_CoreFX}
     XunitLogDir=${__LogDir}/CoreFX
-    if [ ! -d "${XunitLogDir}" ]; then
-      mkdir ${XunitLogDir}
+
+    # Clean up existing logs
+    if [ -d "${XunitLogDir}" ]; then
+        rm -rf "${XunitLogDir}"
     fi
+    mkdir ${XunitLogDir}
 
     pushd ${CoreRT_TestRoot}/CoreFX/runtest