Staging Commit - Netci.groovy refactor and addition of a CoreFX CI job (#5817)

* Return success directly when calling runtests with /corefx flag enabled

* Rename normal scenario to coreclr and fix build failures

netci.groovy

@@ -8,96 +8,152 @@ def project = GithubProject
 // The input branch name (e.g. master)
 def branch = GithubBranchName
 
-def imageVersionMap = ['Windows_NT':'latest-or-auto',
+class Constants {
-                       'OSX10.12':'latest-or-auto',
+
-                       'Ubuntu':'20170118']
+    def static imageVersionMap = ['Windows_NT':'latest-or-auto',
- 
+                           'OSX10.12':'latest-or-auto',
-// Innerloop build OS's
+                           'Ubuntu':'20170118']
-def osList = ['Ubuntu', 'OSX10.12', 'Windows_NT']
+
+    def static scenarios = ['coreclr', 'corefx']
+    
+    // Innerloop build OS's
+    def static osList = ['Ubuntu', 'OSX10.12', 'Windows_NT']
+
+}
 
 // Generate the builds for debug and release, commit and PRJob
-[true, false].each { isPR -> // Defines a closure over true and false, value assigned to isPR
+Constants.scenarios.each { scenario ->
-    ['Debug', 'Release'].each { configuration ->
+    [true, false].each { isPR -> // Defines a closure over true and false, value assigned to isPR
-        osList.each { os ->
+        ['Debug', 'Release'].each { configuration ->
+            Constants.osList.each { os ->
 
-            // Define build string
+                if ((configuration == 'Release' || os != 'Windows_NT') && scenario == 'corefx') {
-            def lowercaseConfiguration = configuration.toLowerCase()
+                    return
-            
+                }
-            // Determine the name for the new job.  The first parameter is the project,
-            // the second parameter is the base name for the job, and the last parameter
-            // is a boolean indicating whether the job will be a PR job.  If true, the
-            // suffix _prtest will be appended.
-            def newJobName = Utilities.getFullJobName(project, lowercaseConfiguration + '_' + os.toLowerCase(), isPR)
-            def buildString = "";
-            def prJobDescription = "${os} ${configuration}";
-            if (configuration == 'Debug') {
-                prJobDescription += " and CoreCLR tests"
-            }
-            
-            // Calculate the build commands
-            if (os == 'Windows_NT') {
-                buildString = "build.cmd ${lowercaseConfiguration}"
-                testScriptString = "tests\\runtest.cmd ${configuration} /coreclr "
-            }
-            else {
-                buildString = "./build.sh ${lowercaseConfiguration}"
-                testScriptString = "tests/runtest.sh ${configuration} -coredumps -coreclr "
-            }
 
-            // Create a new job with the specified name.  The brace opens a new closure
+                // Define build string
-            // and calls made within that closure apply to the newly created job.
+                def lowercaseConfiguration = configuration.toLowerCase()
-            def newJob = job(newJobName) {
-                // This opens the set of build steps that will be run.
-                steps {
-                    if (os == 'Windows_NT') {
-                        // Indicates that a batch script should be run with the build string (see above)
-                        batchFile(buildString)
-                        batchFile("tests\\runtest.cmd ${configuration} /multimodule")
 
-                        if (configuration == 'Debug') {
+                // Determine the name for the new job.  The first parameter is the project,
-                            if (isPR) {
+                // the second parameter is the base name for the job, and the last parameter
-                                // Run a small set of BVTs during PR validation
+                // is a boolean indicating whether the job will be a PR job.  If true, the
-                                batchFile(testScriptString + "Top200")
+                // suffix _prtest will be appended.
-                            }
+                def newJobName = Utilities.getFullJobName(project, lowercaseConfiguration + '_' + os.toLowerCase(), isPR)
-                            else {
+                def buildString = "";
-                                // Run the full set of known passing tests in the post-commit job
+                def prJobDescription = "${os} ${configuration}";
-                                batchFile(testScriptString + "KnownGood /multimodule")
+                if (configuration == 'Debug') {
+                    if (scenario == 'coreclr') {
+                        prJobDescription += " and CoreCLR tests"
+                    }
+                    if (scenario == 'corefx') {
+                        prJobDescription += " and CoreFX tests"
+                    }
+                }
+                
+                def buildCommands = calculateBuildCommands(os, configuration, scenario, isPR)
+
+                // Create a new job with the specified name.  The brace opens a new closure
+                // and calls made within that closure apply to the newly created job.
+                def newJob = job(newJobName) {
+                    // This opens the set of build steps that will be run.
+                    steps {
+                        if (os == 'Windows_NT') {
+                        // Indicates that a batch script should be run with each build command
+                            buildCommands.each { buildCommand -> 
+                                batchFile(buildCommand) 
                             }
                         }
-                    }
+                        else {
-                    else {
+                            buildCommands.each { buildCommand -> 
-                        shell(buildString)
+                                shell(buildCommand)
-
-                        if (configuration == 'Debug') {
-                            if (isPR) {
-                                // Run a small set of BVTs during PR validation
-                                shell(testScriptString + "top200")
-                            }
-                            else {
-                                // Run the full set of known passing tests in the post-commit job
-
-                                // Todo: Enable push test jobs once we establish a reasonable passing set of tests
-                                // shell(testScriptString + "KnownGood")
                             }
                         }
                     }
                 }
-            }
 
-            // This call performs test run checks for the CI.
+                // This call performs test run checks for the CI.
-            Utilities.addXUnitDotNETResults(newJob, '**/testResults.xml')
+                Utilities.addXUnitDotNETResults(newJob, '**/testResults.xml')
-            Utilities.setMachineAffinity(newJob, os, imageVersionMap[os])
+                Utilities.setMachineAffinity(newJob, os, Constants.imageVersionMap[os])
-            Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
+                Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
-            if (isPR) {
+                if (isPR) {
-                Utilities.addGithubPRTriggerForBranch(newJob, branch, prJobDescription)
+                    Utilities.addGithubPRTriggerForBranch(newJob, branch, prJobDescription)
-            }
+                }
-            else {
+                else {
-                // Set a large timeout since the default (2 hours) is insufficient
+                    // Set a large timeout since the default (2 hours) is insufficient
-                Utilities.setJobTimeout(newJob, 1440)
+                    Utilities.setJobTimeout(newJob, 1440)
-                Utilities.addGithubPushTrigger(newJob)
+                    Utilities.addGithubPushTrigger(newJob)
+                }
             }
         }
     }
 }
 
+def static calculateBuildCommands(def os, def configuration, def scenario, def isPR) {
+    
+    def buildCommands = []
+    def lowercaseConfiguration = configuration.toLowerCase()
+    def testScriptString= ''
+
+    if (os == 'Windows_NT') {
+        // Calculate the build commands
+        buildCommands += "build.cmd ${lowercaseConfiguration}"
+
+        // Calculate the test commands
+        buildCommands += "tests\\runtest.cmd ${configuration} /multimodule"
+        if (configuration == 'Debug')
+        {
+            if (scenario == 'coreclr'){
+                testScriptString = "tests\\runtest.cmd ${configuration} /coreclr "
+                if (isPR) {
+                    // Run a small set of BVTs during PR validation
+                    buildCommands += testScriptString + "Top200"
+                }
+                else {
+                    // Run the full set of known passing tests in the post-commit job
+                    buildCommands += testScriptString + "KnownGood /multimodule"
+                }
+            }
+            else if (scenario == 'corefx')
+            {
+                testScriptString = "tests\\runtest.cmd ${configuration} /corefx "
+                
+                //Todo: Add json config files for different testing scenarios
+                buildCommands += testScriptString 
+            }
+        }
+    }
+    else {
+        // Calculate the build commands        
+        buildCommands += "./build.sh ${lowercaseConfiguration}"
+        
+        // Calculate the test commands
+        if (configuration == 'Debug')
+        {
+            if (scenario == 'coreclr')
+            {
+                testScriptString = "tests/runtest.sh ${configuration} -coredumps -coreclr "
+                if (isPR) {
+                    // Run a small set of BVTs during PR validation
+                    buildCommands += testScriptString + "top200"
+                }
+                else {
+                    // Run the full set of known passing tests in the post-commit job
+
+                    // Todo: Enable push test jobs once we establish a reasonable passing set of tests
+                    // shell(testScriptString + "KnownGood")
+                }
+            }
+            else if (scenario == 'corefx')
+            {
+                testScriptString = "tests/runtest.sh ${configuration} -corefx "
+                
+                //Todo: Add json config files for different testing scenarios
+                buildCommands += testScriptString                 
+            }
+        }
+    }
+
+    return buildCommands
+}
+
 JobReport.Report.generateJobReport(out)

tests/runtest.cmd

@@ -49,7 +49,7 @@ if /i "%1" == "/coreclr"  (
 :ExtRepoTestsOk
     goto ArgLoop
 )
-if /i "%1" == "/corefx" (set CoreRT_RunCoreFXTests=true&shift&goto ArgLoop)
+if /i "%1" == "/corefx" exit /b 0
 if /i "%1" == "/coreclrsingletest" (set CoreRT_RunCoreCLRTests=true&set CoreRT_CoreCLRTest=%2&shift&shift&goto ArgLoop)
 if /i "%1" == "/mode" (set CoreRT_TestCompileMode=%2&shift&shift&goto ArgLoop)
 if /i "%1" == "/test" (set CoreRT_TestName=%2&shift&shift&goto ArgLoop)

tests/runtest.sh

@@ -432,6 +432,7 @@ if [ ${CoreRT_RunCoreCLRTests} ]; then
 fi
 
 if [ ${CoreRT_RunCoreFXTests} ]; then
+    exit 0
     run_corefx_tests 
     exit $?
 fi