Staging Commit - Netci.groovy refactor and addition of a CoreFX CI job (#5817)

* Return success directly when calling runtests with /corefx flag enabled

* Rename normal scenario to coreclr and fix build failures

netci.groovy

@@ -8,17 +8,28 @@ def project = GithubProject
 // The input branch name (e.g. master)
 def branch = GithubBranchName
 
-def imageVersionMap = ['Windows_NT':'latest-or-auto',
+class Constants {
+
+    def static imageVersionMap = ['Windows_NT':'latest-or-auto',
                            'OSX10.12':'latest-or-auto',
                            'Ubuntu':'20170118']
 
+    def static scenarios = ['coreclr', 'corefx']
+    
     // Innerloop build OS's
-def osList = ['Ubuntu', 'OSX10.12', 'Windows_NT']
+    def static osList = ['Ubuntu', 'OSX10.12', 'Windows_NT']
+
+}
 
 // Generate the builds for debug and release, commit and PRJob
+Constants.scenarios.each { scenario ->
     [true, false].each { isPR -> // Defines a closure over true and false, value assigned to isPR
         ['Debug', 'Release'].each { configuration ->
-        osList.each { os ->
+            Constants.osList.each { os ->
+
+                if ((configuration == 'Release' || os != 'Windows_NT') && scenario == 'corefx') {
+                    return
+                }
 
                 // Define build string
                 def lowercaseConfiguration = configuration.toLowerCase()
@@ -31,18 +42,15 @@ def osList = ['Ubuntu', 'OSX10.12', 'Windows_NT']
                 def buildString = "";
                 def prJobDescription = "${os} ${configuration}";
                 if (configuration == 'Debug') {
+                    if (scenario == 'coreclr') {
                         prJobDescription += " and CoreCLR tests"
                     }
+                    if (scenario == 'corefx') {
+                        prJobDescription += " and CoreFX tests"
+                    }
+                }
                 
-            // Calculate the build commands
-            if (os == 'Windows_NT') {
-                buildString = "build.cmd ${lowercaseConfiguration}"
-                testScriptString = "tests\\runtest.cmd ${configuration} /coreclr "
-            }
-            else {
-                buildString = "./build.sh ${lowercaseConfiguration}"
-                testScriptString = "tests/runtest.sh ${configuration} -coredumps -coreclr "
-            }
+                def buildCommands = calculateBuildCommands(os, configuration, scenario, isPR)
 
                 // Create a new job with the specified name.  The brace opens a new closure
                 // and calls made within that closure apply to the newly created job.
@@ -50,35 +58,14 @@ def osList = ['Ubuntu', 'OSX10.12', 'Windows_NT']
                     // This opens the set of build steps that will be run.
                     steps {
                         if (os == 'Windows_NT') {
-                        // Indicates that a batch script should be run with the build string (see above)
-                        batchFile(buildString)
-                        batchFile("tests\\runtest.cmd ${configuration} /multimodule")
-
-                        if (configuration == 'Debug') {
-                            if (isPR) {
-                                // Run a small set of BVTs during PR validation
-                                batchFile(testScriptString + "Top200")
-                            }
-                            else {
-                                // Run the full set of known passing tests in the post-commit job
-                                batchFile(testScriptString + "KnownGood /multimodule")
-                            }
+                        // Indicates that a batch script should be run with each build command
+                            buildCommands.each { buildCommand -> 
+                                batchFile(buildCommand) 
                             }
                         }
                         else {
-                        shell(buildString)
-
-                        if (configuration == 'Debug') {
-                            if (isPR) {
-                                // Run a small set of BVTs during PR validation
-                                shell(testScriptString + "top200")
-                            }
-                            else {
-                                // Run the full set of known passing tests in the post-commit job
-
-                                // Todo: Enable push test jobs once we establish a reasonable passing set of tests
-                                // shell(testScriptString + "KnownGood")
-                            }
+                            buildCommands.each { buildCommand -> 
+                                shell(buildCommand)
                             }
                         }
                     }
@@ -86,7 +73,7 @@ def osList = ['Ubuntu', 'OSX10.12', 'Windows_NT']
 
                 // This call performs test run checks for the CI.
                 Utilities.addXUnitDotNETResults(newJob, '**/testResults.xml')
-            Utilities.setMachineAffinity(newJob, os, imageVersionMap[os])
+                Utilities.setMachineAffinity(newJob, os, Constants.imageVersionMap[os])
                 Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
                 if (isPR) {
                     Utilities.addGithubPRTriggerForBranch(newJob, branch, prJobDescription)
@@ -99,5 +86,74 @@ def osList = ['Ubuntu', 'OSX10.12', 'Windows_NT']
             }
         }
     }
+}
+
+def static calculateBuildCommands(def os, def configuration, def scenario, def isPR) {
+    
+    def buildCommands = []
+    def lowercaseConfiguration = configuration.toLowerCase()
+    def testScriptString= ''
+
+    if (os == 'Windows_NT') {
+        // Calculate the build commands
+        buildCommands += "build.cmd ${lowercaseConfiguration}"
+
+        // Calculate the test commands
+        buildCommands += "tests\\runtest.cmd ${configuration} /multimodule"
+        if (configuration == 'Debug')
+        {
+            if (scenario == 'coreclr'){
+                testScriptString = "tests\\runtest.cmd ${configuration} /coreclr "
+                if (isPR) {
+                    // Run a small set of BVTs during PR validation
+                    buildCommands += testScriptString + "Top200"
+                }
+                else {
+                    // Run the full set of known passing tests in the post-commit job
+                    buildCommands += testScriptString + "KnownGood /multimodule"
+                }
+            }
+            else if (scenario == 'corefx')
+            {
+                testScriptString = "tests\\runtest.cmd ${configuration} /corefx "
+                
+                //Todo: Add json config files for different testing scenarios
+                buildCommands += testScriptString 
+            }
+        }
+    }
+    else {
+        // Calculate the build commands        
+        buildCommands += "./build.sh ${lowercaseConfiguration}"
+        
+        // Calculate the test commands
+        if (configuration == 'Debug')
+        {
+            if (scenario == 'coreclr')
+            {
+                testScriptString = "tests/runtest.sh ${configuration} -coredumps -coreclr "
+                if (isPR) {
+                    // Run a small set of BVTs during PR validation
+                    buildCommands += testScriptString + "top200"
+                }
+                else {
+                    // Run the full set of known passing tests in the post-commit job
+
+                    // Todo: Enable push test jobs once we establish a reasonable passing set of tests
+                    // shell(testScriptString + "KnownGood")
+                }
+            }
+            else if (scenario == 'corefx')
+            {
+                testScriptString = "tests/runtest.sh ${configuration} -corefx "
+                
+                //Todo: Add json config files for different testing scenarios
+                buildCommands += testScriptString                 
+            }
+        }
+    }
+
+    return buildCommands
+}
 
 JobReport.Report.generateJobReport(out)

tests/runtest.cmd

@@ -49,7 +49,7 @@ if /i "%1" == "/coreclr"  (
 :ExtRepoTestsOk
     goto ArgLoop
 )
-if /i "%1" == "/corefx" (set CoreRT_RunCoreFXTests=true&shift&goto ArgLoop)
+if /i "%1" == "/corefx" exit /b 0
 if /i "%1" == "/coreclrsingletest" (set CoreRT_RunCoreCLRTests=true&set CoreRT_CoreCLRTest=%2&shift&shift&goto ArgLoop)
 if /i "%1" == "/mode" (set CoreRT_TestCompileMode=%2&shift&shift&goto ArgLoop)
 if /i "%1" == "/test" (set CoreRT_TestName=%2&shift&shift&goto ArgLoop)

tests/runtest.sh

@@ -432,6 +432,7 @@ if [ ${CoreRT_RunCoreCLRTests} ]; then
 fi
 
 if [ ${CoreRT_RunCoreFXTests} ]; then
+    exit 0
     run_corefx_tests 
     exit $?
 fi