From 2953cf1fed60e14c5b9d02cef6fef77fc2f1e0fd Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 18 May 2022 10:50:19 -0400 Subject: [PATCH] [Xharness] Move to use a flag to decide what tests to use (#15032) Co-authored-by: Rolf Bjarne Kvinge --- tests/xharness/Jenkins/Jenkins.cs | 52 ++--- .../Jenkins/MacTestTasksEnumerable.cs | 4 +- .../Jenkins/MakeTestTaskEnumerable.cs | 4 +- .../Jenkins/NUnitTestTasksEnumerable.cs | 14 +- .../xharness/Jenkins/RunDeviceTasksFactory.cs | 14 +- .../Jenkins/RunSimulatorTasksFactory.cs | 10 +- tests/xharness/Jenkins/TestSelector.cs | 205 +++++++++++------- .../xharness/Jenkins/TestVariationsFactory.cs | 20 +- tests/xharness/TestLabel.cs | 102 +++++++++ tests/xharness/xharness.csproj | 1 + 10 files changed, 275 insertions(+), 151 deletions(-) create mode 100644 tests/xharness/TestLabel.cs diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 664c57bb94..dce053fd33 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -21,6 +21,7 @@ namespace Xharness.Jenkins { public readonly IHardwareDeviceLoader Devices; readonly IMlaunchProcessManager processManager; public ITunnelBore TunnelBore { get; private set; } + public TestSelection TestSelection { get; } = new (); readonly TestSelector testSelector; readonly TestVariationsFactory testVariationsFactory; public JenkinsDeviceLoader DeviceLoader { get; private set; } @@ -34,34 +35,7 @@ namespace Xharness.Jenkins { public bool Populating { get; private set; } = true; public IHarness Harness { get; } - public bool IncludeAll; - public bool IncludeBcl; - public bool IncludeMac = true; - public bool IncludeiOS = true; - public bool IncludeiOS64 = true; - public bool IncludeiOS32 = false; // broken in xcode 12 beta 3, not possible with DTK - public bool IncludeiOSExtensions; public bool ForceExtensionBuildOnly; - public bool IncludetvOS = true; - public bool IncludewatchOS = true; - public bool IncludeMmpTest; - public bool IncludeMSBuild = true; - public bool IncludeMtouch; - public bool IncludeBtouch; - public bool IncludeMacBindingProject; - public bool IncludeSimulator = true; - public bool IncludeOldSimulatorTests; - public bool IncludeDevice; - public bool IncludeXtro; - public bool IncludeCecil; - public bool IncludeDocs; - public bool IncludeBCLxUnit; - public bool IncludeBCLNUnit; - public bool IncludeMscorlib; - public bool IncludeNonMonotouch = true; - public bool IncludeMonotouch = true; - public bool IncludeDotNet = true; - public bool IncludeMacCatalyst = true; public bool CleanSuccessfulTestRuns = true; public bool UninstallTestApp = true; @@ -102,7 +76,7 @@ namespace Xharness.Jenkins { Harness = harness ?? throw new ArgumentNullException (nameof (harness)); Simulators = new SimulatorLoader (processManager, new SimulatorSelector ()); Devices = new HardwareDeviceLoader (processManager); - testSelector = new TestSelector (this, processManager, new GitHub (harness, processManager)); + testSelector = new TestSelector (this, new GitHub (harness, processManager)); testVariationsFactory = new TestVariationsFactory (this, processManager); DeviceLoader = new JenkinsDeviceLoader (Simulators, Devices, Logs); resourceManager = new ResourceManager (); @@ -120,16 +94,16 @@ namespace Xharness.Jenkins { if (project.IsBclTest ()) { if (!project.IsBclxUnit ()) - return IncludeBcl || IncludeBCLNUnit; + return TestSelection.IsEnabled (TestLabel.Bcl) || TestSelection.IsEnabled (TestLabel.BclNUnit); if (project.IsMscorlib ()) - return IncludeMscorlib; - return IncludeBcl || IncludeBCLxUnit; + return TestSelection.IsEnabled(TestLabel.Mscorlib); + return TestSelection.IsEnabled (TestLabel.Bcl) || TestSelection.IsEnabled (TestLabel.BclXUnit); } - if (!IncludeMonotouch && project.IsMonotouch ()) + if (!TestSelection.IsEnabled (TestLabel.Monotouch) && project.IsMonotouch ()) return false; - if (!IncludeNonMonotouch && !project.IsMonotouch ()) + if (!TestSelection.IsEnabled (TestLabel.NonMonotouch) && !project.IsMonotouch ()) return false; if (Harness.IncludeSystemPermissionTests == false && project.Name == "introspection") @@ -145,7 +119,7 @@ namespace Xharness.Jenkins { // Missing: // api-diff - testSelector.SelectTests (); + testSelector.SelectTests (TestSelection); DeviceLoader.LoadAllAsync ().DoNotAwait (); @@ -182,7 +156,7 @@ namespace Xharness.Jenkins { TestName = "Xtro", Target = "wrench", WorkingDirectory = Path.Combine (HarnessConfiguration.RootDirectory, "xtro-sharpie"), - Ignored = !IncludeXtro, + Ignored = !TestSelection.IsEnabled (TestLabel.Xtro), Timeout = TimeSpan.FromMinutes (15), SupportsParallelExecution = false, }; @@ -202,7 +176,7 @@ namespace Xharness.Jenkins { TestName = "Xtro", Target = "dotnet-wrench", WorkingDirectory = Path.Combine (HarnessConfiguration.RootDirectory, "xtro-sharpie"), - Ignored = !(IncludeXtro && IncludeDotNet), + Ignored = !(TestSelection.IsEnabled (TestLabel.Xtro) && TestSelection.IsEnabled (TestLabel.Dotnet)), Timeout = TimeSpan.FromMinutes (15), SupportsParallelExecution = false, }; @@ -231,7 +205,7 @@ namespace Xharness.Jenkins { Platform = TestPlatform.iOS, TestName = "Generator tests", Mode = ".NET", - Ignored = !IncludeBtouch, + Ignored = !TestSelection.IsEnabled (TestLabel.Btouch), }; Tasks.Add (runDotNetGenerator); @@ -242,14 +216,14 @@ namespace Xharness.Jenkins { SpecifyPlatform = false, Platform = TestPlatform.All, ProjectConfiguration = "Debug", - Ignored = !IncludeDotNet, + Ignored = !TestSelection.IsEnabled (TestLabel.Dotnet), }; var runDotNetTests = new DotNetTestTask (this, buildDotNetTests, processManager) { TestProject = buildDotNetTestsProject, Platform = TestPlatform.All, TestName = "DotNet tests", Timeout = TimeSpan.FromMinutes (240), - Ignored = !IncludeDotNet, + Ignored = !TestSelection.IsEnabled (TestLabel.Dotnet), }; Tasks.Add (runDotNetTests); diff --git a/tests/xharness/Jenkins/MacTestTasksEnumerable.cs b/tests/xharness/Jenkins/MacTestTasksEnumerable.cs index c4b9949950..cadb59b822 100644 --- a/tests/xharness/Jenkins/MacTestTasksEnumerable.cs +++ b/tests/xharness/Jenkins/MacTestTasksEnumerable.cs @@ -65,11 +65,11 @@ namespace Xharness.Jenkins { { foreach (var project in jenkins.Harness.MacTestProjects) { - bool ignored = !jenkins.IncludeMac; + bool ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac); if (project.Ignore == true) ignored = true; - if (!jenkins.IncludeMmpTest && project.Path.Contains ("mmptest")) + if (!jenkins.TestSelection.IsEnabled (TestLabel.Mmp) && project.Path.Contains ("mmptest")) ignored = true; if (!jenkins.IsIncluded (project)) diff --git a/tests/xharness/Jenkins/MakeTestTaskEnumerable.cs b/tests/xharness/Jenkins/MakeTestTaskEnumerable.cs index 05dd3e2009..f432606db8 100644 --- a/tests/xharness/Jenkins/MakeTestTaskEnumerable.cs +++ b/tests/xharness/Jenkins/MakeTestTaskEnumerable.cs @@ -25,7 +25,7 @@ namespace Xharness.Jenkins { TestName = "MMP Regression Tests", Target = "all", // -j" + Environment.ProcessorCount, WorkingDirectory = Path.Combine (HarnessConfiguration.RootDirectory, "mmp-regression"), - Ignored = !jenkins.IncludeMmpTest || !jenkins.IncludeMac, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mmp) || !jenkins.TestSelection.IsEnabled (TestLabel.Mac), Timeout = TimeSpan.FromMinutes (30), SupportsParallelExecution = false, // Already doing parallel execution by running "make -jX" }; @@ -41,7 +41,7 @@ namespace Xharness.Jenkins { TestName = "Mac Binding Projects", Target = "all", WorkingDirectory = Path.Combine (HarnessConfiguration.RootDirectory, "mac-binding-project"), - Ignored = !jenkins.IncludeMacBindingProject || !jenkins.IncludeMac, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.MacBindingProject) || !jenkins.TestSelection.IsEnabled (TestLabel.Mac), Timeout = TimeSpan.FromMinutes (15), }; yield return runMacBindingProject; diff --git a/tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs b/tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs index b15a1b45c9..8aeaf1c8f6 100644 --- a/tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs +++ b/tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs @@ -36,7 +36,7 @@ namespace Xharness.Jenkins { TestName = "MSBuild tests", Mode = "Tasks", Timeout = TimeSpan.FromMinutes (60), - Ignored = !jenkins.IncludeMSBuild, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Msbuild), SupportsParallelExecution = false, }; yield return nunitExecutioniOSMSBuild; @@ -58,7 +58,7 @@ namespace Xharness.Jenkins { TestName = "MSBuild tests", Mode = "Integration", Timeout = TimeSpan.FromMinutes (120), - Ignored = !jenkins.IncludeMSBuild, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Msbuild), SupportsParallelExecution = false, }; yield return nunitExecutioniOSMSBuildIntegration; @@ -77,7 +77,7 @@ namespace Xharness.Jenkins { TestName = "Install Sources tests", Mode = "iOS", Timeout = TimeSpan.FromMinutes (60), - Ignored = !jenkins.IncludeMac && !jenkins.IncludeSimulator, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac) && !jenkins.TestSelection.IsEnabled (TestLabel.iOSSimulator), }; yield return nunitExecutionInstallSource; @@ -95,7 +95,7 @@ namespace Xharness.Jenkins { Platform = TestPlatform.iOS, TestName = "MTouch tests", Timeout = TimeSpan.FromMinutes (180), - Ignored = !jenkins.IncludeMtouch, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mtouch), InProcess = true, }; yield return nunitExecutionMTouch; @@ -115,7 +115,7 @@ namespace Xharness.Jenkins { TestName = "Generator tests", Mode = "NUnit", Timeout = TimeSpan.FromMinutes (10), - Ignored = !jenkins.IncludeBtouch, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Btouch), }; yield return runGenerator; @@ -125,7 +125,7 @@ namespace Xharness.Jenkins { SpecifyPlatform = false, Platform = TestPlatform.All, ProjectConfiguration = "Debug", - Ignored = !jenkins.IncludeCecil, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Cecil), }; var runCecilTests = new NUnitExecuteTask (jenkins, buildCecilTests, processManager) { TestLibrary = Path.Combine (Path.GetDirectoryName (buildCecilTestsProject.Path), "bin", "Debug", "net472", "cecil-tests.dll"), @@ -133,7 +133,7 @@ namespace Xharness.Jenkins { Platform = TestPlatform.iOS, TestName = "Cecil-based tests", Timeout = TimeSpan.FromMinutes (5), - Ignored = !jenkins.IncludeCecil, + Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Cecil), InProcess = true, }; yield return runCecilTests; diff --git a/tests/xharness/Jenkins/RunDeviceTasksFactory.cs b/tests/xharness/Jenkins/RunDeviceTasksFactory.cs index fd96066fbf..4e183b5957 100644 --- a/tests/xharness/Jenkins/RunDeviceTasksFactory.cs +++ b/tests/xharness/Jenkins/RunDeviceTasksFactory.cs @@ -20,7 +20,7 @@ namespace Xharness.Jenkins { if (project.SkipDeviceVariations) continue; - bool ignored = project.Ignore ?? !jenkins.IncludeDevice; + bool ignored = project.Ignore ?? !jenkins.TestSelection.IsEnabled (TestLabel.Device); if (!jenkins.IsIncluded (project)) ignored = true; if (project.IsDotNetProject) @@ -61,7 +61,7 @@ namespace Xharness.Jenkins { tunnelBore: jenkins.TunnelBore, errorKnowledgeBase: jenkins.ErrorKnowledgeBase, useTcpTunnel: jenkins.Harness.UseTcpTunnel, - candidates: jenkins.Devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludeiOS64 }); + candidates: jenkins.Devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.iOS64) }); var build32 = new MSBuildTask (jenkins: jenkins, testProject: project, processManager: processManager) { ProjectConfiguration = project.Name != "dont link" ? "Debug32" : "Release32", @@ -78,7 +78,7 @@ namespace Xharness.Jenkins { tunnelBore: jenkins.TunnelBore, errorKnowledgeBase: jenkins.ErrorKnowledgeBase, useTcpTunnel: jenkins.Harness.UseTcpTunnel, - candidates: jenkins.Devices.Connected32BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludeiOS32 }); + candidates: jenkins.Devices.Connected32BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.iOS32)}); if (createTodayExtension) { var todayProject = project.GenerateVariations ? project.AsTodayExtensionProject () : project; @@ -97,7 +97,7 @@ namespace Xharness.Jenkins { tunnelBore: jenkins.TunnelBore, errorKnowledgeBase: jenkins.ErrorKnowledgeBase, useTcpTunnel: jenkins.Harness.UseTcpTunnel, - candidates: jenkins.Devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludeiOSExtensions, BuildOnly = jenkins.ForceExtensionBuildOnly }); + candidates: jenkins.Devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled(TestLabel.iOSExtension), BuildOnly = jenkins.ForceExtensionBuildOnly }); } } @@ -118,7 +118,7 @@ namespace Xharness.Jenkins { tunnelBore: jenkins.TunnelBore, errorKnowledgeBase: jenkins.ErrorKnowledgeBase, useTcpTunnel: jenkins.Harness.UseTcpTunnel, - candidates: jenkins.Devices.ConnectedTV.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludetvOS }); + candidates: jenkins.Devices.ConnectedTV.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.tvOS) }); } if (createwatchOS) { @@ -139,7 +139,7 @@ namespace Xharness.Jenkins { tunnelBore: jenkins.TunnelBore, errorKnowledgeBase: jenkins.ErrorKnowledgeBase, useTcpTunnel: jenkins.Harness.UseTcpTunnel, - candidates: jenkins.Devices.ConnectedWatch) { Ignored = !jenkins.IncludewatchOS }); + candidates: jenkins.Devices.ConnectedWatch) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.watchOS) }); } if (!project.SkipwatchOSARM64_32Variation) { @@ -158,7 +158,7 @@ namespace Xharness.Jenkins { tunnelBore: jenkins.TunnelBore, errorKnowledgeBase: jenkins.ErrorKnowledgeBase, useTcpTunnel: jenkins.Harness.UseTcpTunnel, - candidates: jenkins.Devices.ConnectedWatch32_64.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludewatchOS }); + candidates: jenkins.Devices.ConnectedWatch32_64.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.watchOS) }); } } foreach (var task in projectTasks) { diff --git a/tests/xharness/Jenkins/RunSimulatorTasksFactory.cs b/tests/xharness/Jenkins/RunSimulatorTasksFactory.cs index 0a8569b34f..e5b118c5d1 100644 --- a/tests/xharness/Jenkins/RunSimulatorTasksFactory.cs +++ b/tests/xharness/Jenkins/RunSimulatorTasksFactory.cs @@ -20,7 +20,7 @@ namespace Xharness.Jenkins { if (!project.IsExecutableProject) continue; - bool ignored = project.Ignore ?? !jenkins.IncludeSimulator; + bool ignored = project.Ignore ?? !jenkins.TestSelection.IsEnabled (TestLabel.iOSSimulator); if (!jenkins.IsIncluded (project)) ignored = true; @@ -48,20 +48,20 @@ namespace Xharness.Jenkins { switch (testPlatform) { case TestPlatform.iOS_Unified: case TestPlatform.iOS_TodayExtension64: - configIgnored |= !jenkins.IncludeiOS64; + configIgnored |= !jenkins.TestSelection.IsEnabled (TestLabel.iOS64); break; case TestPlatform.tvOS: - configIgnored |= !jenkins.IncludetvOS; + configIgnored |= !jenkins.TestSelection.IsEnabled (TestLabel.tvOS); break; case TestPlatform.watchOS: - configIgnored |= !jenkins.IncludewatchOS; + configIgnored |= !jenkins.TestSelection.IsEnabled (TestLabel.watchOS); break; default: Console.WriteLine ("Unknown test platform for ignore check: {0}", testPlatform); break; } - configIgnored |= project.IsDotNetProject && !jenkins.IncludeDotNet; + configIgnored |= project.IsDotNetProject && !jenkins.TestSelection.IsEnabled (TestLabel.Dotnet); var derived = new MSBuildTask (jenkins: jenkins, testProject: project, processManager: processManager); derived.ProjectConfiguration = config; diff --git a/tests/xharness/Jenkins/TestSelector.cs b/tests/xharness/Jenkins/TestSelector.cs index c215f42a30..aa16718c71 100644 --- a/tests/xharness/Jenkins/TestSelector.cs +++ b/tests/xharness/Jenkins/TestSelector.cs @@ -2,14 +2,46 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Eventing.Reader; using System.IO; using System.Linq; using System.Text.RegularExpressions; -using Microsoft.DotNet.XHarness.Common.Execution; using Microsoft.DotNet.XHarness.Common.Logging; namespace Xharness.Jenkins { + class TestSelection { + TestLabel selection = + TestLabel.None | + TestLabel.tvOS | + TestLabel.watchOS | + TestLabel.Msbuild | + TestLabel.iOSSimulator | + TestLabel.Monotouch | + TestLabel.NonMonotouch | + TestLabel.Dotnet | + TestLabel.MacCatalyst; + + public bool ForceExtensionBuildOnly { get; set; } + + public void SetEnabled (TestLabel label, bool enable) + { + if (enable) { + selection |= label; + } else { + selection &= ~label; + } + } + + public void SetEnabled (string label, bool value) + { + var testLabel = label.GetLabel (); + SetEnabled (testLabel, value); + } + + public bool IsEnabled (TestLabel label) + => selection.HasFlag (label); + } /// /// Allows to select the tests to be ran depending on certain conditions such as labels of modified files. /// @@ -18,7 +50,6 @@ namespace Xharness.Jenkins { #region private vars readonly Jenkins jenkins; - readonly IProcessManager processManager; readonly IVersionControlSystem vcs; ILog MainLog => jenkins.MainLog; @@ -93,10 +124,9 @@ namespace Xharness.Jenkins { #endregion - public TestSelector (Jenkins jenkins, IProcessManager processManager, IVersionControlSystem versionControlSystem) + public TestSelector (Jenkins jenkins, IVersionControlSystem versionControlSystem) { this.jenkins = jenkins; - this.processManager = processManager; this.vcs = versionControlSystem; } @@ -107,7 +137,7 @@ namespace Xharness.Jenkins { } // 'filenames' is a list of filename prefixes, unless the name has a star character, in which case it's interpreted as a regex expression. - void SetEnabled (IEnumerable files, string [] filenames, string testname, ref bool value) + void SetEnabled (IEnumerable files, string [] filenames, string testname, TestSelection selection) { MainLog.WriteLine ($"Checking if test {testname} should be enabled according to the modified files."); @@ -127,11 +157,13 @@ namespace Xharness.Jenkins { for (var i = 0; i < filenames.Length; i++) { var prefix = filenames [i]; if (file.StartsWith (prefix, StringComparison.Ordinal)) { - value = true; + selection.SetEnabled (testname, true); MainLog.WriteLine ("Enabled '{0}' tests because the modified file '{1}' matches prefix '{2}'", testname, file, prefix); return; - } else if (regexes [i]?.IsMatch (file) == true) { - value = true; + } + + if (regexes [i]?.IsMatch (file) == true) { + selection.SetEnabled (testname, true); MainLog.WriteLine ("Enabled '{0}' tests because the modified file '{1}' matches regex '{2}'", testname, file, prefix); return; } @@ -140,34 +172,46 @@ namespace Xharness.Jenkins { } // Returns true if the value was changed. - bool SetEnabled (HashSet labels, string testname, ref bool value) + bool SetEnabled (HashSet labels, string testname, TestSelection selection) { if (labels.Contains ("skip-" + testname + "-tests")) { MainLog.WriteLine ("Disabled '{0}' tests because the label 'skip-{0}-tests' is set.", testname); - if (testname == "ios") - jenkins.IncludeiOS32 = jenkins.IncludeiOS64 = false; - value = false; + if (testname == "ios") { + selection.SetEnabled (TestLabel.iOS64, false); + selection.SetEnabled (TestLabel.iOS32, false); + } + + selection.SetEnabled (testname, false); return true; - } else if (labels.Contains ("run-" + testname + "-tests")) { + } + + if (labels.Contains ("run-" + testname + "-tests")) { MainLog.WriteLine ("Enabled '{0}' tests because the label 'run-{0}-tests' is set.", testname); - if (testname == "ios") - jenkins.IncludeiOS32 = jenkins.IncludeiOS64 = true; - value = true; + if (testname == "ios") { + selection.SetEnabled (TestLabel.iOS64, true); + selection.SetEnabled (TestLabel.iOS32, true); + } + + selection.SetEnabled (testname, true); return true; - } else if (labels.Contains ("skip-all-tests")) { + } + + if (labels.Contains ("skip-all-tests")) { MainLog.WriteLine ("Disabled '{0}' tests because the label 'skip-all-tests' is set.", testname); - value = false; + selection.SetEnabled (testname, false); return true; - } else if (labels.Contains ("run-all-tests")) { + } + + if (labels.Contains ("run-all-tests")) { MainLog.WriteLine ("Enabled '{0}' tests because the label 'run-all-tests' is set.", testname); - value = true; + selection.SetEnabled (testname, true); return true; } // respect any default value return false; } - void SelectTestsByModifiedFiles (int pullRequest) + void SelectTestsByModifiedFiles (int pullRequest, TestSelection selection) { // toArray so that we do not always enumerate all the time. var files = vcs.GetModifiedFiles (pullRequest).ToArray (); @@ -176,19 +220,23 @@ namespace Xharness.Jenkins { foreach (var f in files) MainLog.WriteLine (" {0}", f); - SetEnabled (files, mtouchPrefixes, "mtouch", ref jenkins.IncludeMtouch); - SetEnabled (files, mmpPrefixes, "mmp", ref jenkins.IncludeMmpTest); - SetEnabled (files, bclPrefixes, "bcl", ref jenkins.IncludeBcl); - SetEnabled (files, btouchPrefixes, "btouch", ref jenkins.IncludeBtouch); - SetEnabled (files, macBindingProject, "mac-binding-project", ref jenkins.IncludeMacBindingProject); - SetEnabled (files, xtroPrefixes, "xtro", ref jenkins.IncludeXtro); - SetEnabled (files, cecilPrefixes, "cecil", ref jenkins.IncludeCecil); - SetEnabled (files, dotnetFilenames, "dotnet", ref jenkins.IncludeDotNet); - SetEnabled (files, msbuildFilenames, "msbuild", ref jenkins.IncludeMSBuild); - SetEnabled (files, xharnessPrefix, "all", ref jenkins.IncludeAll); + SetEnabled (files, mtouchPrefixes, "mtouch", selection); + SetEnabled (files, mmpPrefixes, "mmp", selection); + SetEnabled (files, bclPrefixes, "bcl", selection); + SetEnabled (files, btouchPrefixes, "btouch", selection); + SetEnabled (files, macBindingProject, "mac-binding-project", selection); + SetEnabled (files, xtroPrefixes, "xtro", selection); + SetEnabled (files, cecilPrefixes, "cecil", selection); + SetEnabled (files, dotnetFilenames, "dotnet", selection); + SetEnabled (files, msbuildFilenames, "msbuild", selection); + // xharness will run all tests, but we do not want to run the device tests + // if they were not selected + var devicesEnabled = selection.IsEnabled (TestLabel.Device); + SetEnabled (files, xharnessPrefix, "all", selection); + selection.SetEnabled (TestLabel.Device, devicesEnabled); } - void SelectTestsByLabel (int pullRequest) + void SelectTestsByLabel (int pullRequest, TestSelection selection) { var labels = new HashSet (); if (Harness.Labels.Any ()) { @@ -231,72 +279,71 @@ namespace Xharness.Jenkins { MainLog.WriteLine ($"In total found {labels.Count ()} label(s): {string.Join (", ", labels.ToArray ())}"); // disabled by default - SetEnabled (labels, "mtouch", ref jenkins.IncludeMtouch); - SetEnabled (labels, "mmp", ref jenkins.IncludeMmpTest); - SetEnabled (labels, "bcl", ref jenkins.IncludeBcl); - SetEnabled (labels, "bcl-xunit", ref jenkins.IncludeBCLxUnit); - SetEnabled (labels, "bcl-nunit", ref jenkins.IncludeBCLNUnit); - SetEnabled (labels, "mscorlib", ref jenkins.IncludeMscorlib); - SetEnabled (labels, "btouch", ref jenkins.IncludeBtouch); - SetEnabled (labels, "mac-binding-project", ref jenkins.IncludeMacBindingProject); - SetEnabled (labels, "ios-extensions", ref jenkins.IncludeiOSExtensions); - SetEnabled (labels, "device", ref jenkins.IncludeDevice); - SetEnabled (labels, "xtro", ref jenkins.IncludeXtro); - SetEnabled (labels, "cecil", ref jenkins.IncludeCecil); - SetEnabled (labels, "old-simulator", ref jenkins.IncludeOldSimulatorTests); - SetEnabled (labels, "dotnet", ref jenkins.IncludeDotNet); - SetEnabled (labels, "all", ref jenkins.IncludeAll); + SetEnabled (labels, "mtouch", selection); + SetEnabled (labels, "mmp", selection); + SetEnabled (labels, "bcl", selection); + SetEnabled (labels, "bcl-xunit", selection); + SetEnabled (labels, "bcl-nunit", selection); + SetEnabled (labels, "mscorlib", selection); + SetEnabled (labels, "btouch", selection); + SetEnabled (labels, "mac-binding-project", selection); + SetEnabled (labels, "ios-extensions", selection); + SetEnabled (labels, "device", selection); + SetEnabled (labels, "xtro", selection); + SetEnabled (labels, "cecil", selection); + SetEnabled (labels, "old-simulator", selection); + SetEnabled (labels, "dotnet", selection); + SetEnabled (labels, "all", selection); // enabled by default - SetEnabled (labels, "ios-32", ref jenkins.IncludeiOS32); - SetEnabled (labels, "ios-64", ref jenkins.IncludeiOS64); - SetEnabled (labels, "ios", ref jenkins.IncludeiOS); // Needs to be set after `ios-32` and `ios-64` (because it can reset them) - SetEnabled (labels, "tvos", ref jenkins.IncludetvOS); - SetEnabled (labels, "watchos", ref jenkins.IncludewatchOS); - SetEnabled (labels, "mac", ref jenkins.IncludeMac); - SetEnabled (labels, "msbuild", ref jenkins.IncludeMSBuild); - SetEnabled (labels, "ios-simulator", ref jenkins.IncludeSimulator); - SetEnabled (labels, "non-monotouch", ref jenkins.IncludeNonMonotouch); - SetEnabled (labels, "monotouch", ref jenkins.IncludeMonotouch); + SetEnabled (labels, "ios-32", selection); + SetEnabled (labels, "ios-64", selection); + SetEnabled (labels, "ios", selection); + SetEnabled (labels, "tvos", selection); + SetEnabled (labels, "watchos", selection); + SetEnabled (labels, "mac", selection); + SetEnabled (labels, "msbuild", selection); + SetEnabled (labels, "ios-simulator", selection); + SetEnabled (labels, "non-monotouch", selection); + SetEnabled (labels, "monotouch", selection); - bool inc_permission_tests = false; - if (SetEnabled (labels, "system-permission", ref inc_permission_tests)) - Harness.IncludeSystemPermissionTests = inc_permission_tests; + if (SetEnabled (labels, "system-permission", selection)) + Harness.IncludeSystemPermissionTests = selection.IsEnabled (TestLabel.SystemPermission); // docs is a bit special: // - can only be executed if the Xamarin-specific parts of the build is enabled // - enabled by default if the current branch is main (or, for a pull request, if the target branch is main) - var changed = SetEnabled (labels, "docs", ref jenkins.IncludeDocs); + var changed = SetEnabled (labels, "docs", selection); if (Harness.ENABLE_XAMARIN) { if (!changed) { // don't override any value set using labels var branchName = Environment.GetEnvironmentVariable ("BRANCH_NAME"); if (!string.IsNullOrEmpty (branchName)) { - jenkins.IncludeDocs = branchName == "main"; - if (jenkins.IncludeDocs) + selection.SetEnabled (TestLabel.Docs, branchName == "main"); + if (selection.IsEnabled (TestLabel.Docs)) MainLog.WriteLine ("Enabled 'docs' tests because the current branch is 'main'."); } else if (pullRequest > 0) { - jenkins.IncludeDocs = vcs.GetPullRequestTargetBranch (pullRequest) == "main"; - if (jenkins.IncludeDocs) + selection.SetEnabled (TestLabel.Docs, vcs.GetPullRequestTargetBranch (pullRequest) == "main"); + if (selection.IsEnabled (TestLabel.Docs)) MainLog.WriteLine ("Enabled 'docs' tests because the target branch is 'main'."); } } } else { - if (jenkins.IncludeDocs) { - jenkins.IncludeDocs = false; // could have been enabled by 'run-all-tests', so disable it if we can't run it. + if (selection.IsEnabled (TestLabel.Docs)) { + selection.SetEnabled (TestLabel.Docs, false); // could have been enabled by 'run-all-tests', so disable it if we can't run it. MainLog.WriteLine ("Disabled 'docs' tests because the Xamarin-specific parts of the build are not enabled."); } } // old simulator tests is also a bit special: // - enabled by default if using a beta Xcode, otherwise disabled by default - changed = SetEnabled (labels, "old-simulator", ref jenkins.IncludeOldSimulatorTests); + changed = SetEnabled (labels, "old-simulator", selection); if (!changed && jenkins.IsBetaXcode) { - jenkins.IncludeOldSimulatorTests = true; + selection.SetEnabled (TestLabel.OldiOSSimulator, true); MainLog.WriteLine ("Enabled 'old-simulator' tests because we're using a beta Xcode."); } } - public void SelectTests () + public void SelectTests (TestSelection selection) { if (!int.TryParse (Environment.GetEnvironmentVariable ("PR_ID"), out int pullRequest)) MainLog.WriteLine ("The environment variable 'PR_ID' was not found, so no pull requests will be checked for test selection."); @@ -304,39 +351,39 @@ namespace Xharness.Jenkins { // First check if can auto-select any tests based on which files were modified. // This will only enable additional tests, never disable tests. if (pullRequest > 0) - SelectTestsByModifiedFiles (pullRequest); + SelectTestsByModifiedFiles (pullRequest, selection); // Then we check for labels. Labels are manually set, so those override // whatever we did automatically. - SelectTestsByLabel (pullRequest); + SelectTestsByLabel (pullRequest, selection); DisableKnownFailingDeviceTests (); if (!Harness.INCLUDE_IOS) { MainLog.WriteLine ("The iOS build is disabled, so any iOS tests will be disabled as well."); - jenkins.IncludeiOS = false; - jenkins.IncludeiOS64 = false; - jenkins.IncludeiOS32 = false; + selection.SetEnabled (TestLabel.iOS, false); + selection.SetEnabled (TestLabel.iOS64, false); + selection.SetEnabled (TestLabel.iOS32, false); } if (!Harness.INCLUDE_WATCH) { MainLog.WriteLine ("The watchOS build is disabled, so any watchOS tests will be disabled as well."); - jenkins.IncludewatchOS = false; + selection.SetEnabled (TestLabel.watchOS, false); } if (!Harness.INCLUDE_TVOS) { MainLog.WriteLine ("The tvOS build is disabled, so any tvOS tests will be disabled as well."); - jenkins.IncludetvOS = false; + selection.SetEnabled (TestLabel.tvOS, false); } if (!Harness.INCLUDE_MAC) { MainLog.WriteLine ("The macOS build is disabled, so any macOS tests will be disabled as well."); - jenkins.IncludeMac = false; + selection.SetEnabled (TestLabel.Mac, false); } if (!Harness.ENABLE_DOTNET) { MainLog.WriteLine ("The .NET build is disabled, so any .NET tests will be disabled as well."); - jenkins.IncludeDotNet = false; + selection.SetEnabled (TestLabel.Dotnet, false); } } } diff --git a/tests/xharness/Jenkins/TestVariationsFactory.cs b/tests/xharness/Jenkins/TestVariationsFactory.cs index fb159165a6..dbb42c79a2 100644 --- a/tests/xharness/Jenkins/TestVariationsFactory.cs +++ b/tests/xharness/Jenkins/TestVariationsFactory.cs @@ -122,7 +122,7 @@ namespace Xharness.Jenkins { yield return new TestData { Variation = "Debug (LinkSdk)", Debug = true, Profiling = false, LinkMode = test.TestProject.IsDotNetProject ? "SdkOnly" : "LinkSdk", Ignored = ignore }; yield return new TestData { Variation = "Debug (static registrar)", MTouchExtraArgs = "--registrar:static", Debug = true, Profiling = false, Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore }; yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore }; - yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore ?? !jenkins.IncludeAll }; + yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore ?? !jenkins.TestSelection.IsEnabled (TestLabel.All) }; if (test.TestProject.IsDotNetProject && mac_supports_arm64) yield return new TestData { Variation = "Debug (ARM64)", Debug = true, Profiling = false, Ignored = !mac_supports_arm64 ? true : ignore, RuntimeIdentifier = arm64_sim_runtime_identifier, }; @@ -136,7 +136,7 @@ namespace Xharness.Jenkins { Variation = $"Debug ({test.Platform.GetSimulatorMinVersion ()})", Debug = true, Candidates = jenkins.Simulators.SelectDevices (target.GetTargetOs (true), jenkins.SimulatorLoadLog, true), - Ignored = ignore ?? !jenkins.IncludeOldSimulatorTests, + Ignored = ignore ?? !jenkins.TestSelection.IsEnabled (TestLabel.OldiOSSimulator), }; break; } @@ -146,24 +146,24 @@ namespace Xharness.Jenkins { switch (test.TestName) { case "monotouch-test": if (test.TestProject.IsDotNetProject) { - yield return new TestData { Variation = "Debug (ARM64)", Debug = true, Profiling = false, Ignored = !jenkins.IncludeMac || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier, }; + yield return new TestData { Variation = "Debug (ARM64)", Debug = true, Profiling = false, Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac) || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier, }; if (test.Platform != TestPlatform.MacCatalyst) { - yield return new TestData { Variation = "Debug (static registrar)", MonoBundlingExtraArgs = "--registrar:static", Debug = true, Undefines = "DYNAMIC_REGISTRAR", Ignored = !jenkins.IncludeMac, }; - yield return new TestData { Variation = "Debug (static registrar, ARM64)", MonoBundlingExtraArgs = "--registrar:static", Debug = true, Undefines = "DYNAMIC_REGISTRAR", Profiling = false, Ignored = !jenkins.IncludeMac || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier, }; + yield return new TestData { Variation = "Debug (static registrar)", MonoBundlingExtraArgs = "--registrar:static", Debug = true, Undefines = "DYNAMIC_REGISTRAR", Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac), }; + yield return new TestData { Variation = "Debug (static registrar, ARM64)", MonoBundlingExtraArgs = "--registrar:static", Debug = true, Undefines = "DYNAMIC_REGISTRAR", Profiling = false, Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac) || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier, }; } if (test.Platform == TestPlatform.MacCatalyst) - yield return new TestData { Variation = "Release (ARM64, LLVM)", Debug = false, UseLlvm = true, Ignored = !jenkins.IncludeMacCatalyst || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier }; + yield return new TestData { Variation = "Release (ARM64, LLVM)", Debug = false, UseLlvm = true, Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.MacCatalyst) || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier }; } break; case "xammac tests": switch (test.ProjectConfiguration) { case "Release": yield return new TestData { Variation = "Release (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all", Debug = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR" }; - yield return new TestData { Variation = "Release (ARM64)", XamMacArch = "ARM64", Debug = false, Ignored = !mac_supports_arm64 || !jenkins.IncludeMac }; + yield return new TestData { Variation = "Release (ARM64)", XamMacArch = "ARM64", Debug = false, Ignored = !mac_supports_arm64 || !jenkins.TestSelection.IsEnabled (TestLabel.Mac) }; break; case "Debug": - yield return new TestData { Variation = "Debug (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = !(jenkins.IncludeAll && jenkins.IncludeMac) }; - yield return new TestData { Variation = "Debug (ARM64)", XamMacArch = "ARM64", Debug = true, Ignored = !mac_supports_arm64 || !jenkins.IncludeMac }; + yield return new TestData { Variation = "Debug (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = !(jenkins.TestSelection.IsEnabled (TestLabel.All) && jenkins.TestSelection.IsEnabled (TestLabel.Mac)) }; + yield return new TestData { Variation = "Debug (ARM64)", XamMacArch = "ARM64", Debug = true, Ignored = !mac_supports_arm64 || !jenkins.TestSelection.IsEnabled (TestLabel.Mac) }; break; } break; @@ -267,7 +267,7 @@ namespace Xharness.Jenkins { clone.Xml.Save (clone.Path); }); - ignored |= clone.IsDotNetProject && !jenkins.IncludeDotNet; + ignored |= clone.IsDotNetProject && !jenkins.TestSelection.IsEnabled (TestLabel.Dotnet); var build = new MSBuildTask (jenkins: jenkins, testProject: clone, processManager: processManager); build.ProjectConfiguration = configuration; diff --git a/tests/xharness/TestLabel.cs b/tests/xharness/TestLabel.cs new file mode 100644 index 0000000000..45b01e7945 --- /dev/null +++ b/tests/xharness/TestLabel.cs @@ -0,0 +1,102 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using Mono.Unix.Native; + +#nullable enable + +namespace Xharness { + [AttributeUsage (AttributeTargets.Field)] + public class LabelAttribute : Attribute { + public string Label { get; } + + public LabelAttribute (string label) + { + Label = label; + } + } + + [Flags] + public enum TestLabel : ulong { + [Label ("none")] + None = 0, + [Label ("bcl")] + Bcl = 1 << 1, + [Label ("bcl-nunit")] + BclNUnit = 1 << 2, + [Label ("bcl-xunit")] + BclXUnit = 1 << 3, + [Label ("btouch")] + Btouch = 1 << 4, + [Label ("cecil")] + Cecil = 1 << 5, + [Label ("device")] + Device = 1 << 6, + [Label ("docs")] + Docs = 1 << 7, + [Label ("dotnet")] + Dotnet = 1 << 8, + [Label ("ios")] + iOS = 1 << 9, + [Label ("ios-extensions")] + iOSExtension = 1 << 10, + [Label ("ios-simulator")] + iOSSimulator = 1 << 11, + [Label ("ios-32")] + iOS32 = 1 << 12, + [Label ("ios-64")] + iOS64 = 1 << 13, + [Label ("mac")] + Mac = 1 << 14, + [Label ("mac-binding-project")] + MacBindingProject = 1 << 15, + [Label ("maccatalyst")] + MacCatalyst = 1 << 16, + [Label ("mmp")] + Mmp = 1 << 17, + [Label ("mscorlib")] + Mscorlib = 1 << 18, + [Label ("monotouch")] + Monotouch = 1 << 19, + [Label ("msbuild")] + Msbuild = 1 << 20, + [Label ("mtouch")] + Mtouch = 1 << 21, + [Label ("non-monotouch")] + NonMonotouch = 1 << 22, + [Label ("old-simulator")] + OldiOSSimulator = 1 << 23, + [Label ("system-permission")] + SystemPermission = 1 << 24, + [Label ("tvos")] + tvOS = 1 << 15, + [Label ("watchos")] + watchOS = 1 << 26, + [Label ("xtro")] + Xtro = 1 << 27, + [Label ("all")] + All = 0xFFFFFFFF, + } + + static class TestLabelExtensions { + public static string GetLabel (this TestLabel self) + { + var enumType = typeof(TestLabel); + var name = Enum.GetName (typeof (TestLabel), self); + var attr = enumType.GetField (name).GetCustomAttribute (); + return attr.Label; + } + + public static TestLabel GetLabel (this string self) + { + foreach (var obj in Enum.GetValues (typeof (TestLabel))) { + if (obj is TestLabel value && value.GetLabel () == self) { + return value; + } + } + + throw new InvalidOperationException ($"Unknown label '{self}'"); + } + } +} diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index 7b58b48e02..c04d685240 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -161,6 +161,7 @@ +