[Xharness] Move to use a flag to decide what tests to use (#15032)

Co-authored-by: Rolf Bjarne Kvinge <rolf@xamarin.com>
This commit is contained in:
Manuel de la Pena 2022-05-18 10:50:19 -04:00 коммит произвёл GitHub
Родитель 873cc1ea96
Коммит 2953cf1fed
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
10 изменённых файлов: 275 добавлений и 151 удалений

Просмотреть файл

@ -21,6 +21,7 @@ namespace Xharness.Jenkins {
public readonly IHardwareDeviceLoader Devices;
readonly IMlaunchProcessManager processManager;
public ITunnelBore TunnelBore { get; private set; }
public TestSelection TestSelection { get; } = new ();
readonly TestSelector testSelector;
readonly TestVariationsFactory testVariationsFactory;
public JenkinsDeviceLoader DeviceLoader { get; private set; }
@ -34,34 +35,7 @@ namespace Xharness.Jenkins {
public bool Populating { get; private set; } = true;
public IHarness Harness { get; }
public bool IncludeAll;
public bool IncludeBcl;
public bool IncludeMac = true;
public bool IncludeiOS = true;
public bool IncludeiOS64 = true;
public bool IncludeiOS32 = false; // broken in xcode 12 beta 3, not possible with DTK
public bool IncludeiOSExtensions;
public bool ForceExtensionBuildOnly;
public bool IncludetvOS = true;
public bool IncludewatchOS = true;
public bool IncludeMmpTest;
public bool IncludeMSBuild = true;
public bool IncludeMtouch;
public bool IncludeBtouch;
public bool IncludeMacBindingProject;
public bool IncludeSimulator = true;
public bool IncludeOldSimulatorTests;
public bool IncludeDevice;
public bool IncludeXtro;
public bool IncludeCecil;
public bool IncludeDocs;
public bool IncludeBCLxUnit;
public bool IncludeBCLNUnit;
public bool IncludeMscorlib;
public bool IncludeNonMonotouch = true;
public bool IncludeMonotouch = true;
public bool IncludeDotNet = true;
public bool IncludeMacCatalyst = true;
public bool CleanSuccessfulTestRuns = true;
public bool UninstallTestApp = true;
@ -102,7 +76,7 @@ namespace Xharness.Jenkins {
Harness = harness ?? throw new ArgumentNullException (nameof (harness));
Simulators = new SimulatorLoader (processManager, new SimulatorSelector ());
Devices = new HardwareDeviceLoader (processManager);
testSelector = new TestSelector (this, processManager, new GitHub (harness, processManager));
testSelector = new TestSelector (this, new GitHub (harness, processManager));
testVariationsFactory = new TestVariationsFactory (this, processManager);
DeviceLoader = new JenkinsDeviceLoader (Simulators, Devices, Logs);
resourceManager = new ResourceManager ();
@ -120,16 +94,16 @@ namespace Xharness.Jenkins {
if (project.IsBclTest ()) {
if (!project.IsBclxUnit ())
return IncludeBcl || IncludeBCLNUnit;
return TestSelection.IsEnabled (TestLabel.Bcl) || TestSelection.IsEnabled (TestLabel.BclNUnit);
if (project.IsMscorlib ())
return IncludeMscorlib;
return IncludeBcl || IncludeBCLxUnit;
return TestSelection.IsEnabled(TestLabel.Mscorlib);
return TestSelection.IsEnabled (TestLabel.Bcl) || TestSelection.IsEnabled (TestLabel.BclXUnit);
}
if (!IncludeMonotouch && project.IsMonotouch ())
if (!TestSelection.IsEnabled (TestLabel.Monotouch) && project.IsMonotouch ())
return false;
if (!IncludeNonMonotouch && !project.IsMonotouch ())
if (!TestSelection.IsEnabled (TestLabel.NonMonotouch) && !project.IsMonotouch ())
return false;
if (Harness.IncludeSystemPermissionTests == false && project.Name == "introspection")
@ -145,7 +119,7 @@ namespace Xharness.Jenkins {
// Missing:
// api-diff
testSelector.SelectTests ();
testSelector.SelectTests (TestSelection);
DeviceLoader.LoadAllAsync ().DoNotAwait ();
@ -182,7 +156,7 @@ namespace Xharness.Jenkins {
TestName = "Xtro",
Target = "wrench",
WorkingDirectory = Path.Combine (HarnessConfiguration.RootDirectory, "xtro-sharpie"),
Ignored = !IncludeXtro,
Ignored = !TestSelection.IsEnabled (TestLabel.Xtro),
Timeout = TimeSpan.FromMinutes (15),
SupportsParallelExecution = false,
};
@ -202,7 +176,7 @@ namespace Xharness.Jenkins {
TestName = "Xtro",
Target = "dotnet-wrench",
WorkingDirectory = Path.Combine (HarnessConfiguration.RootDirectory, "xtro-sharpie"),
Ignored = !(IncludeXtro && IncludeDotNet),
Ignored = !(TestSelection.IsEnabled (TestLabel.Xtro) && TestSelection.IsEnabled (TestLabel.Dotnet)),
Timeout = TimeSpan.FromMinutes (15),
SupportsParallelExecution = false,
};
@ -231,7 +205,7 @@ namespace Xharness.Jenkins {
Platform = TestPlatform.iOS,
TestName = "Generator tests",
Mode = ".NET",
Ignored = !IncludeBtouch,
Ignored = !TestSelection.IsEnabled (TestLabel.Btouch),
};
Tasks.Add (runDotNetGenerator);
@ -242,14 +216,14 @@ namespace Xharness.Jenkins {
SpecifyPlatform = false,
Platform = TestPlatform.All,
ProjectConfiguration = "Debug",
Ignored = !IncludeDotNet,
Ignored = !TestSelection.IsEnabled (TestLabel.Dotnet),
};
var runDotNetTests = new DotNetTestTask (this, buildDotNetTests, processManager) {
TestProject = buildDotNetTestsProject,
Platform = TestPlatform.All,
TestName = "DotNet tests",
Timeout = TimeSpan.FromMinutes (240),
Ignored = !IncludeDotNet,
Ignored = !TestSelection.IsEnabled (TestLabel.Dotnet),
};
Tasks.Add (runDotNetTests);

Просмотреть файл

@ -65,11 +65,11 @@ namespace Xharness.Jenkins {
{
foreach (var project in jenkins.Harness.MacTestProjects) {
bool ignored = !jenkins.IncludeMac;
bool ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac);
if (project.Ignore == true)
ignored = true;
if (!jenkins.IncludeMmpTest && project.Path.Contains ("mmptest"))
if (!jenkins.TestSelection.IsEnabled (TestLabel.Mmp) && project.Path.Contains ("mmptest"))
ignored = true;
if (!jenkins.IsIncluded (project))

Просмотреть файл

@ -25,7 +25,7 @@ namespace Xharness.Jenkins {
TestName = "MMP Regression Tests",
Target = "all", // -j" + Environment.ProcessorCount,
WorkingDirectory = Path.Combine (HarnessConfiguration.RootDirectory, "mmp-regression"),
Ignored = !jenkins.IncludeMmpTest || !jenkins.IncludeMac,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mmp) || !jenkins.TestSelection.IsEnabled (TestLabel.Mac),
Timeout = TimeSpan.FromMinutes (30),
SupportsParallelExecution = false, // Already doing parallel execution by running "make -jX"
};
@ -41,7 +41,7 @@ namespace Xharness.Jenkins {
TestName = "Mac Binding Projects",
Target = "all",
WorkingDirectory = Path.Combine (HarnessConfiguration.RootDirectory, "mac-binding-project"),
Ignored = !jenkins.IncludeMacBindingProject || !jenkins.IncludeMac,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.MacBindingProject) || !jenkins.TestSelection.IsEnabled (TestLabel.Mac),
Timeout = TimeSpan.FromMinutes (15),
};
yield return runMacBindingProject;

Просмотреть файл

@ -36,7 +36,7 @@ namespace Xharness.Jenkins {
TestName = "MSBuild tests",
Mode = "Tasks",
Timeout = TimeSpan.FromMinutes (60),
Ignored = !jenkins.IncludeMSBuild,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Msbuild),
SupportsParallelExecution = false,
};
yield return nunitExecutioniOSMSBuild;
@ -58,7 +58,7 @@ namespace Xharness.Jenkins {
TestName = "MSBuild tests",
Mode = "Integration",
Timeout = TimeSpan.FromMinutes (120),
Ignored = !jenkins.IncludeMSBuild,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Msbuild),
SupportsParallelExecution = false,
};
yield return nunitExecutioniOSMSBuildIntegration;
@ -77,7 +77,7 @@ namespace Xharness.Jenkins {
TestName = "Install Sources tests",
Mode = "iOS",
Timeout = TimeSpan.FromMinutes (60),
Ignored = !jenkins.IncludeMac && !jenkins.IncludeSimulator,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac) && !jenkins.TestSelection.IsEnabled (TestLabel.iOSSimulator),
};
yield return nunitExecutionInstallSource;
@ -95,7 +95,7 @@ namespace Xharness.Jenkins {
Platform = TestPlatform.iOS,
TestName = "MTouch tests",
Timeout = TimeSpan.FromMinutes (180),
Ignored = !jenkins.IncludeMtouch,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mtouch),
InProcess = true,
};
yield return nunitExecutionMTouch;
@ -115,7 +115,7 @@ namespace Xharness.Jenkins {
TestName = "Generator tests",
Mode = "NUnit",
Timeout = TimeSpan.FromMinutes (10),
Ignored = !jenkins.IncludeBtouch,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Btouch),
};
yield return runGenerator;
@ -125,7 +125,7 @@ namespace Xharness.Jenkins {
SpecifyPlatform = false,
Platform = TestPlatform.All,
ProjectConfiguration = "Debug",
Ignored = !jenkins.IncludeCecil,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Cecil),
};
var runCecilTests = new NUnitExecuteTask (jenkins, buildCecilTests, processManager) {
TestLibrary = Path.Combine (Path.GetDirectoryName (buildCecilTestsProject.Path), "bin", "Debug", "net472", "cecil-tests.dll"),
@ -133,7 +133,7 @@ namespace Xharness.Jenkins {
Platform = TestPlatform.iOS,
TestName = "Cecil-based tests",
Timeout = TimeSpan.FromMinutes (5),
Ignored = !jenkins.IncludeCecil,
Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Cecil),
InProcess = true,
};
yield return runCecilTests;

Просмотреть файл

@ -20,7 +20,7 @@ namespace Xharness.Jenkins {
if (project.SkipDeviceVariations)
continue;
bool ignored = project.Ignore ?? !jenkins.IncludeDevice;
bool ignored = project.Ignore ?? !jenkins.TestSelection.IsEnabled (TestLabel.Device);
if (!jenkins.IsIncluded (project))
ignored = true;
if (project.IsDotNetProject)
@ -61,7 +61,7 @@ namespace Xharness.Jenkins {
tunnelBore: jenkins.TunnelBore,
errorKnowledgeBase: jenkins.ErrorKnowledgeBase,
useTcpTunnel: jenkins.Harness.UseTcpTunnel,
candidates: jenkins.Devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludeiOS64 });
candidates: jenkins.Devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.iOS64) });
var build32 = new MSBuildTask (jenkins: jenkins, testProject: project, processManager: processManager) {
ProjectConfiguration = project.Name != "dont link" ? "Debug32" : "Release32",
@ -78,7 +78,7 @@ namespace Xharness.Jenkins {
tunnelBore: jenkins.TunnelBore,
errorKnowledgeBase: jenkins.ErrorKnowledgeBase,
useTcpTunnel: jenkins.Harness.UseTcpTunnel,
candidates: jenkins.Devices.Connected32BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludeiOS32 });
candidates: jenkins.Devices.Connected32BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.iOS32)});
if (createTodayExtension) {
var todayProject = project.GenerateVariations ? project.AsTodayExtensionProject () : project;
@ -97,7 +97,7 @@ namespace Xharness.Jenkins {
tunnelBore: jenkins.TunnelBore,
errorKnowledgeBase: jenkins.ErrorKnowledgeBase,
useTcpTunnel: jenkins.Harness.UseTcpTunnel,
candidates: jenkins.Devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludeiOSExtensions, BuildOnly = jenkins.ForceExtensionBuildOnly });
candidates: jenkins.Devices.Connected64BitIOS.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled(TestLabel.iOSExtension), BuildOnly = jenkins.ForceExtensionBuildOnly });
}
}
@ -118,7 +118,7 @@ namespace Xharness.Jenkins {
tunnelBore: jenkins.TunnelBore,
errorKnowledgeBase: jenkins.ErrorKnowledgeBase,
useTcpTunnel: jenkins.Harness.UseTcpTunnel,
candidates: jenkins.Devices.ConnectedTV.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludetvOS });
candidates: jenkins.Devices.ConnectedTV.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.tvOS) });
}
if (createwatchOS) {
@ -139,7 +139,7 @@ namespace Xharness.Jenkins {
tunnelBore: jenkins.TunnelBore,
errorKnowledgeBase: jenkins.ErrorKnowledgeBase,
useTcpTunnel: jenkins.Harness.UseTcpTunnel,
candidates: jenkins.Devices.ConnectedWatch) { Ignored = !jenkins.IncludewatchOS });
candidates: jenkins.Devices.ConnectedWatch) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.watchOS) });
}
if (!project.SkipwatchOSARM64_32Variation) {
@ -158,7 +158,7 @@ namespace Xharness.Jenkins {
tunnelBore: jenkins.TunnelBore,
errorKnowledgeBase: jenkins.ErrorKnowledgeBase,
useTcpTunnel: jenkins.Harness.UseTcpTunnel,
candidates: jenkins.Devices.ConnectedWatch32_64.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.IncludewatchOS });
candidates: jenkins.Devices.ConnectedWatch32_64.Where (d => project.IsSupported (d.DevicePlatform, d.ProductVersion))) { Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.watchOS) });
}
}
foreach (var task in projectTasks) {

Просмотреть файл

@ -20,7 +20,7 @@ namespace Xharness.Jenkins {
if (!project.IsExecutableProject)
continue;
bool ignored = project.Ignore ?? !jenkins.IncludeSimulator;
bool ignored = project.Ignore ?? !jenkins.TestSelection.IsEnabled (TestLabel.iOSSimulator);
if (!jenkins.IsIncluded (project))
ignored = true;
@ -48,20 +48,20 @@ namespace Xharness.Jenkins {
switch (testPlatform) {
case TestPlatform.iOS_Unified:
case TestPlatform.iOS_TodayExtension64:
configIgnored |= !jenkins.IncludeiOS64;
configIgnored |= !jenkins.TestSelection.IsEnabled (TestLabel.iOS64);
break;
case TestPlatform.tvOS:
configIgnored |= !jenkins.IncludetvOS;
configIgnored |= !jenkins.TestSelection.IsEnabled (TestLabel.tvOS);
break;
case TestPlatform.watchOS:
configIgnored |= !jenkins.IncludewatchOS;
configIgnored |= !jenkins.TestSelection.IsEnabled (TestLabel.watchOS);
break;
default:
Console.WriteLine ("Unknown test platform for ignore check: {0}", testPlatform);
break;
}
configIgnored |= project.IsDotNetProject && !jenkins.IncludeDotNet;
configIgnored |= project.IsDotNetProject && !jenkins.TestSelection.IsEnabled (TestLabel.Dotnet);
var derived = new MSBuildTask (jenkins: jenkins, testProject: project, processManager: processManager);
derived.ProjectConfiguration = config;

Просмотреть файл

@ -2,14 +2,46 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Eventing.Reader;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.DotNet.XHarness.Common.Execution;
using Microsoft.DotNet.XHarness.Common.Logging;
namespace Xharness.Jenkins {
class TestSelection {
TestLabel selection =
TestLabel.None |
TestLabel.tvOS |
TestLabel.watchOS |
TestLabel.Msbuild |
TestLabel.iOSSimulator |
TestLabel.Monotouch |
TestLabel.NonMonotouch |
TestLabel.Dotnet |
TestLabel.MacCatalyst;
public bool ForceExtensionBuildOnly { get; set; }
public void SetEnabled (TestLabel label, bool enable)
{
if (enable) {
selection |= label;
} else {
selection &= ~label;
}
}
public void SetEnabled (string label, bool value)
{
var testLabel = label.GetLabel ();
SetEnabled (testLabel, value);
}
public bool IsEnabled (TestLabel label)
=> selection.HasFlag (label);
}
/// <summary>
/// Allows to select the tests to be ran depending on certain conditions such as labels of modified files.
/// </summary>
@ -18,7 +50,6 @@ namespace Xharness.Jenkins {
#region private vars
readonly Jenkins jenkins;
readonly IProcessManager processManager;
readonly IVersionControlSystem vcs;
ILog MainLog => jenkins.MainLog;
@ -93,10 +124,9 @@ namespace Xharness.Jenkins {
#endregion
public TestSelector (Jenkins jenkins, IProcessManager processManager, IVersionControlSystem versionControlSystem)
public TestSelector (Jenkins jenkins, IVersionControlSystem versionControlSystem)
{
this.jenkins = jenkins;
this.processManager = processManager;
this.vcs = versionControlSystem;
}
@ -107,7 +137,7 @@ namespace Xharness.Jenkins {
}
// 'filenames' is a list of filename prefixes, unless the name has a star character, in which case it's interpreted as a regex expression.
void SetEnabled (IEnumerable<string> files, string [] filenames, string testname, ref bool value)
void SetEnabled (IEnumerable<string> files, string [] filenames, string testname, TestSelection selection)
{
MainLog.WriteLine ($"Checking if test {testname} should be enabled according to the modified files.");
@ -127,11 +157,13 @@ namespace Xharness.Jenkins {
for (var i = 0; i < filenames.Length; i++) {
var prefix = filenames [i];
if (file.StartsWith (prefix, StringComparison.Ordinal)) {
value = true;
selection.SetEnabled (testname, true);
MainLog.WriteLine ("Enabled '{0}' tests because the modified file '{1}' matches prefix '{2}'", testname, file, prefix);
return;
} else if (regexes [i]?.IsMatch (file) == true) {
value = true;
}
if (regexes [i]?.IsMatch (file) == true) {
selection.SetEnabled (testname, true);
MainLog.WriteLine ("Enabled '{0}' tests because the modified file '{1}' matches regex '{2}'", testname, file, prefix);
return;
}
@ -140,34 +172,46 @@ namespace Xharness.Jenkins {
}
// Returns true if the value was changed.
bool SetEnabled (HashSet<string> labels, string testname, ref bool value)
bool SetEnabled (HashSet<string> labels, string testname, TestSelection selection)
{
if (labels.Contains ("skip-" + testname + "-tests")) {
MainLog.WriteLine ("Disabled '{0}' tests because the label 'skip-{0}-tests' is set.", testname);
if (testname == "ios")
jenkins.IncludeiOS32 = jenkins.IncludeiOS64 = false;
value = false;
if (testname == "ios") {
selection.SetEnabled (TestLabel.iOS64, false);
selection.SetEnabled (TestLabel.iOS32, false);
}
selection.SetEnabled (testname, false);
return true;
} else if (labels.Contains ("run-" + testname + "-tests")) {
}
if (labels.Contains ("run-" + testname + "-tests")) {
MainLog.WriteLine ("Enabled '{0}' tests because the label 'run-{0}-tests' is set.", testname);
if (testname == "ios")
jenkins.IncludeiOS32 = jenkins.IncludeiOS64 = true;
value = true;
if (testname == "ios") {
selection.SetEnabled (TestLabel.iOS64, true);
selection.SetEnabled (TestLabel.iOS32, true);
}
selection.SetEnabled (testname, true);
return true;
} else if (labels.Contains ("skip-all-tests")) {
}
if (labels.Contains ("skip-all-tests")) {
MainLog.WriteLine ("Disabled '{0}' tests because the label 'skip-all-tests' is set.", testname);
value = false;
selection.SetEnabled (testname, false);
return true;
} else if (labels.Contains ("run-all-tests")) {
}
if (labels.Contains ("run-all-tests")) {
MainLog.WriteLine ("Enabled '{0}' tests because the label 'run-all-tests' is set.", testname);
value = true;
selection.SetEnabled (testname, true);
return true;
}
// respect any default value
return false;
}
void SelectTestsByModifiedFiles (int pullRequest)
void SelectTestsByModifiedFiles (int pullRequest, TestSelection selection)
{
// toArray so that we do not always enumerate all the time.
var files = vcs.GetModifiedFiles (pullRequest).ToArray ();
@ -176,19 +220,23 @@ namespace Xharness.Jenkins {
foreach (var f in files)
MainLog.WriteLine (" {0}", f);
SetEnabled (files, mtouchPrefixes, "mtouch", ref jenkins.IncludeMtouch);
SetEnabled (files, mmpPrefixes, "mmp", ref jenkins.IncludeMmpTest);
SetEnabled (files, bclPrefixes, "bcl", ref jenkins.IncludeBcl);
SetEnabled (files, btouchPrefixes, "btouch", ref jenkins.IncludeBtouch);
SetEnabled (files, macBindingProject, "mac-binding-project", ref jenkins.IncludeMacBindingProject);
SetEnabled (files, xtroPrefixes, "xtro", ref jenkins.IncludeXtro);
SetEnabled (files, cecilPrefixes, "cecil", ref jenkins.IncludeCecil);
SetEnabled (files, dotnetFilenames, "dotnet", ref jenkins.IncludeDotNet);
SetEnabled (files, msbuildFilenames, "msbuild", ref jenkins.IncludeMSBuild);
SetEnabled (files, xharnessPrefix, "all", ref jenkins.IncludeAll);
SetEnabled (files, mtouchPrefixes, "mtouch", selection);
SetEnabled (files, mmpPrefixes, "mmp", selection);
SetEnabled (files, bclPrefixes, "bcl", selection);
SetEnabled (files, btouchPrefixes, "btouch", selection);
SetEnabled (files, macBindingProject, "mac-binding-project", selection);
SetEnabled (files, xtroPrefixes, "xtro", selection);
SetEnabled (files, cecilPrefixes, "cecil", selection);
SetEnabled (files, dotnetFilenames, "dotnet", selection);
SetEnabled (files, msbuildFilenames, "msbuild", selection);
// xharness will run all tests, but we do not want to run the device tests
// if they were not selected
var devicesEnabled = selection.IsEnabled (TestLabel.Device);
SetEnabled (files, xharnessPrefix, "all", selection);
selection.SetEnabled (TestLabel.Device, devicesEnabled);
}
void SelectTestsByLabel (int pullRequest)
void SelectTestsByLabel (int pullRequest, TestSelection selection)
{
var labels = new HashSet<string> ();
if (Harness.Labels.Any ()) {
@ -231,72 +279,71 @@ namespace Xharness.Jenkins {
MainLog.WriteLine ($"In total found {labels.Count ()} label(s): {string.Join (", ", labels.ToArray ())}");
// disabled by default
SetEnabled (labels, "mtouch", ref jenkins.IncludeMtouch);
SetEnabled (labels, "mmp", ref jenkins.IncludeMmpTest);
SetEnabled (labels, "bcl", ref jenkins.IncludeBcl);
SetEnabled (labels, "bcl-xunit", ref jenkins.IncludeBCLxUnit);
SetEnabled (labels, "bcl-nunit", ref jenkins.IncludeBCLNUnit);
SetEnabled (labels, "mscorlib", ref jenkins.IncludeMscorlib);
SetEnabled (labels, "btouch", ref jenkins.IncludeBtouch);
SetEnabled (labels, "mac-binding-project", ref jenkins.IncludeMacBindingProject);
SetEnabled (labels, "ios-extensions", ref jenkins.IncludeiOSExtensions);
SetEnabled (labels, "device", ref jenkins.IncludeDevice);
SetEnabled (labels, "xtro", ref jenkins.IncludeXtro);
SetEnabled (labels, "cecil", ref jenkins.IncludeCecil);
SetEnabled (labels, "old-simulator", ref jenkins.IncludeOldSimulatorTests);
SetEnabled (labels, "dotnet", ref jenkins.IncludeDotNet);
SetEnabled (labels, "all", ref jenkins.IncludeAll);
SetEnabled (labels, "mtouch", selection);
SetEnabled (labels, "mmp", selection);
SetEnabled (labels, "bcl", selection);
SetEnabled (labels, "bcl-xunit", selection);
SetEnabled (labels, "bcl-nunit", selection);
SetEnabled (labels, "mscorlib", selection);
SetEnabled (labels, "btouch", selection);
SetEnabled (labels, "mac-binding-project", selection);
SetEnabled (labels, "ios-extensions", selection);
SetEnabled (labels, "device", selection);
SetEnabled (labels, "xtro", selection);
SetEnabled (labels, "cecil", selection);
SetEnabled (labels, "old-simulator", selection);
SetEnabled (labels, "dotnet", selection);
SetEnabled (labels, "all", selection);
// enabled by default
SetEnabled (labels, "ios-32", ref jenkins.IncludeiOS32);
SetEnabled (labels, "ios-64", ref jenkins.IncludeiOS64);
SetEnabled (labels, "ios", ref jenkins.IncludeiOS); // Needs to be set after `ios-32` and `ios-64` (because it can reset them)
SetEnabled (labels, "tvos", ref jenkins.IncludetvOS);
SetEnabled (labels, "watchos", ref jenkins.IncludewatchOS);
SetEnabled (labels, "mac", ref jenkins.IncludeMac);
SetEnabled (labels, "msbuild", ref jenkins.IncludeMSBuild);
SetEnabled (labels, "ios-simulator", ref jenkins.IncludeSimulator);
SetEnabled (labels, "non-monotouch", ref jenkins.IncludeNonMonotouch);
SetEnabled (labels, "monotouch", ref jenkins.IncludeMonotouch);
SetEnabled (labels, "ios-32", selection);
SetEnabled (labels, "ios-64", selection);
SetEnabled (labels, "ios", selection);
SetEnabled (labels, "tvos", selection);
SetEnabled (labels, "watchos", selection);
SetEnabled (labels, "mac", selection);
SetEnabled (labels, "msbuild", selection);
SetEnabled (labels, "ios-simulator", selection);
SetEnabled (labels, "non-monotouch", selection);
SetEnabled (labels, "monotouch", selection);
bool inc_permission_tests = false;
if (SetEnabled (labels, "system-permission", ref inc_permission_tests))
Harness.IncludeSystemPermissionTests = inc_permission_tests;
if (SetEnabled (labels, "system-permission", selection))
Harness.IncludeSystemPermissionTests = selection.IsEnabled (TestLabel.SystemPermission);
// docs is a bit special:
// - can only be executed if the Xamarin-specific parts of the build is enabled
// - enabled by default if the current branch is main (or, for a pull request, if the target branch is main)
var changed = SetEnabled (labels, "docs", ref jenkins.IncludeDocs);
var changed = SetEnabled (labels, "docs", selection);
if (Harness.ENABLE_XAMARIN) {
if (!changed) { // don't override any value set using labels
var branchName = Environment.GetEnvironmentVariable ("BRANCH_NAME");
if (!string.IsNullOrEmpty (branchName)) {
jenkins.IncludeDocs = branchName == "main";
if (jenkins.IncludeDocs)
selection.SetEnabled (TestLabel.Docs, branchName == "main");
if (selection.IsEnabled (TestLabel.Docs))
MainLog.WriteLine ("Enabled 'docs' tests because the current branch is 'main'.");
} else if (pullRequest > 0) {
jenkins.IncludeDocs = vcs.GetPullRequestTargetBranch (pullRequest) == "main";
if (jenkins.IncludeDocs)
selection.SetEnabled (TestLabel.Docs, vcs.GetPullRequestTargetBranch (pullRequest) == "main");
if (selection.IsEnabled (TestLabel.Docs))
MainLog.WriteLine ("Enabled 'docs' tests because the target branch is 'main'.");
}
}
} else {
if (jenkins.IncludeDocs) {
jenkins.IncludeDocs = false; // could have been enabled by 'run-all-tests', so disable it if we can't run it.
if (selection.IsEnabled (TestLabel.Docs)) {
selection.SetEnabled (TestLabel.Docs, false); // could have been enabled by 'run-all-tests', so disable it if we can't run it.
MainLog.WriteLine ("Disabled 'docs' tests because the Xamarin-specific parts of the build are not enabled.");
}
}
// old simulator tests is also a bit special:
// - enabled by default if using a beta Xcode, otherwise disabled by default
changed = SetEnabled (labels, "old-simulator", ref jenkins.IncludeOldSimulatorTests);
changed = SetEnabled (labels, "old-simulator", selection);
if (!changed && jenkins.IsBetaXcode) {
jenkins.IncludeOldSimulatorTests = true;
selection.SetEnabled (TestLabel.OldiOSSimulator, true);
MainLog.WriteLine ("Enabled 'old-simulator' tests because we're using a beta Xcode.");
}
}
public void SelectTests ()
public void SelectTests (TestSelection selection)
{
if (!int.TryParse (Environment.GetEnvironmentVariable ("PR_ID"), out int pullRequest))
MainLog.WriteLine ("The environment variable 'PR_ID' was not found, so no pull requests will be checked for test selection.");
@ -304,39 +351,39 @@ namespace Xharness.Jenkins {
// First check if can auto-select any tests based on which files were modified.
// This will only enable additional tests, never disable tests.
if (pullRequest > 0)
SelectTestsByModifiedFiles (pullRequest);
SelectTestsByModifiedFiles (pullRequest, selection);
// Then we check for labels. Labels are manually set, so those override
// whatever we did automatically.
SelectTestsByLabel (pullRequest);
SelectTestsByLabel (pullRequest, selection);
DisableKnownFailingDeviceTests ();
if (!Harness.INCLUDE_IOS) {
MainLog.WriteLine ("The iOS build is disabled, so any iOS tests will be disabled as well.");
jenkins.IncludeiOS = false;
jenkins.IncludeiOS64 = false;
jenkins.IncludeiOS32 = false;
selection.SetEnabled (TestLabel.iOS, false);
selection.SetEnabled (TestLabel.iOS64, false);
selection.SetEnabled (TestLabel.iOS32, false);
}
if (!Harness.INCLUDE_WATCH) {
MainLog.WriteLine ("The watchOS build is disabled, so any watchOS tests will be disabled as well.");
jenkins.IncludewatchOS = false;
selection.SetEnabled (TestLabel.watchOS, false);
}
if (!Harness.INCLUDE_TVOS) {
MainLog.WriteLine ("The tvOS build is disabled, so any tvOS tests will be disabled as well.");
jenkins.IncludetvOS = false;
selection.SetEnabled (TestLabel.tvOS, false);
}
if (!Harness.INCLUDE_MAC) {
MainLog.WriteLine ("The macOS build is disabled, so any macOS tests will be disabled as well.");
jenkins.IncludeMac = false;
selection.SetEnabled (TestLabel.Mac, false);
}
if (!Harness.ENABLE_DOTNET) {
MainLog.WriteLine ("The .NET build is disabled, so any .NET tests will be disabled as well.");
jenkins.IncludeDotNet = false;
selection.SetEnabled (TestLabel.Dotnet, false);
}
}
}

Просмотреть файл

@ -122,7 +122,7 @@ namespace Xharness.Jenkins {
yield return new TestData { Variation = "Debug (LinkSdk)", Debug = true, Profiling = false, LinkMode = test.TestProject.IsDotNetProject ? "SdkOnly" : "LinkSdk", Ignored = ignore };
yield return new TestData { Variation = "Debug (static registrar)", MTouchExtraArgs = "--registrar:static", Debug = true, Profiling = false, Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore };
yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore };
yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore ?? !jenkins.IncludeAll };
yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore ?? !jenkins.TestSelection.IsEnabled (TestLabel.All) };
if (test.TestProject.IsDotNetProject && mac_supports_arm64)
yield return new TestData { Variation = "Debug (ARM64)", Debug = true, Profiling = false, Ignored = !mac_supports_arm64 ? true : ignore, RuntimeIdentifier = arm64_sim_runtime_identifier, };
@ -136,7 +136,7 @@ namespace Xharness.Jenkins {
Variation = $"Debug ({test.Platform.GetSimulatorMinVersion ()})",
Debug = true,
Candidates = jenkins.Simulators.SelectDevices (target.GetTargetOs (true), jenkins.SimulatorLoadLog, true),
Ignored = ignore ?? !jenkins.IncludeOldSimulatorTests,
Ignored = ignore ?? !jenkins.TestSelection.IsEnabled (TestLabel.OldiOSSimulator),
};
break;
}
@ -146,24 +146,24 @@ namespace Xharness.Jenkins {
switch (test.TestName) {
case "monotouch-test":
if (test.TestProject.IsDotNetProject) {
yield return new TestData { Variation = "Debug (ARM64)", Debug = true, Profiling = false, Ignored = !jenkins.IncludeMac || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier, };
yield return new TestData { Variation = "Debug (ARM64)", Debug = true, Profiling = false, Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac) || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier, };
if (test.Platform != TestPlatform.MacCatalyst) {
yield return new TestData { Variation = "Debug (static registrar)", MonoBundlingExtraArgs = "--registrar:static", Debug = true, Undefines = "DYNAMIC_REGISTRAR", Ignored = !jenkins.IncludeMac, };
yield return new TestData { Variation = "Debug (static registrar, ARM64)", MonoBundlingExtraArgs = "--registrar:static", Debug = true, Undefines = "DYNAMIC_REGISTRAR", Profiling = false, Ignored = !jenkins.IncludeMac || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier, };
yield return new TestData { Variation = "Debug (static registrar)", MonoBundlingExtraArgs = "--registrar:static", Debug = true, Undefines = "DYNAMIC_REGISTRAR", Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac), };
yield return new TestData { Variation = "Debug (static registrar, ARM64)", MonoBundlingExtraArgs = "--registrar:static", Debug = true, Undefines = "DYNAMIC_REGISTRAR", Profiling = false, Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Mac) || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier, };
}
if (test.Platform == TestPlatform.MacCatalyst)
yield return new TestData { Variation = "Release (ARM64, LLVM)", Debug = false, UseLlvm = true, Ignored = !jenkins.IncludeMacCatalyst || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier };
yield return new TestData { Variation = "Release (ARM64, LLVM)", Debug = false, UseLlvm = true, Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.MacCatalyst) || !mac_supports_arm64, RuntimeIdentifier = arm64_runtime_identifier };
}
break;
case "xammac tests":
switch (test.ProjectConfiguration) {
case "Release":
yield return new TestData { Variation = "Release (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all", Debug = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR" };
yield return new TestData { Variation = "Release (ARM64)", XamMacArch = "ARM64", Debug = false, Ignored = !mac_supports_arm64 || !jenkins.IncludeMac };
yield return new TestData { Variation = "Release (ARM64)", XamMacArch = "ARM64", Debug = false, Ignored = !mac_supports_arm64 || !jenkins.TestSelection.IsEnabled (TestLabel.Mac) };
break;
case "Debug":
yield return new TestData { Variation = "Debug (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = !(jenkins.IncludeAll && jenkins.IncludeMac) };
yield return new TestData { Variation = "Debug (ARM64)", XamMacArch = "ARM64", Debug = true, Ignored = !mac_supports_arm64 || !jenkins.IncludeMac };
yield return new TestData { Variation = "Debug (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = !(jenkins.TestSelection.IsEnabled (TestLabel.All) && jenkins.TestSelection.IsEnabled (TestLabel.Mac)) };
yield return new TestData { Variation = "Debug (ARM64)", XamMacArch = "ARM64", Debug = true, Ignored = !mac_supports_arm64 || !jenkins.TestSelection.IsEnabled (TestLabel.Mac) };
break;
}
break;
@ -267,7 +267,7 @@ namespace Xharness.Jenkins {
clone.Xml.Save (clone.Path);
});
ignored |= clone.IsDotNetProject && !jenkins.IncludeDotNet;
ignored |= clone.IsDotNetProject && !jenkins.TestSelection.IsEnabled (TestLabel.Dotnet);
var build = new MSBuildTask (jenkins: jenkins, testProject: clone, processManager: processManager);
build.ProjectConfiguration = configuration;

102
tests/xharness/TestLabel.cs Normal file
Просмотреть файл

@ -0,0 +1,102 @@
using System;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using Mono.Unix.Native;
#nullable enable
namespace Xharness {
[AttributeUsage (AttributeTargets.Field)]
public class LabelAttribute : Attribute {
public string Label { get; }
public LabelAttribute (string label)
{
Label = label;
}
}
[Flags]
public enum TestLabel : ulong {
[Label ("none")]
None = 0,
[Label ("bcl")]
Bcl = 1 << 1,
[Label ("bcl-nunit")]
BclNUnit = 1 << 2,
[Label ("bcl-xunit")]
BclXUnit = 1 << 3,
[Label ("btouch")]
Btouch = 1 << 4,
[Label ("cecil")]
Cecil = 1 << 5,
[Label ("device")]
Device = 1 << 6,
[Label ("docs")]
Docs = 1 << 7,
[Label ("dotnet")]
Dotnet = 1 << 8,
[Label ("ios")]
iOS = 1 << 9,
[Label ("ios-extensions")]
iOSExtension = 1 << 10,
[Label ("ios-simulator")]
iOSSimulator = 1 << 11,
[Label ("ios-32")]
iOS32 = 1 << 12,
[Label ("ios-64")]
iOS64 = 1 << 13,
[Label ("mac")]
Mac = 1 << 14,
[Label ("mac-binding-project")]
MacBindingProject = 1 << 15,
[Label ("maccatalyst")]
MacCatalyst = 1 << 16,
[Label ("mmp")]
Mmp = 1 << 17,
[Label ("mscorlib")]
Mscorlib = 1 << 18,
[Label ("monotouch")]
Monotouch = 1 << 19,
[Label ("msbuild")]
Msbuild = 1 << 20,
[Label ("mtouch")]
Mtouch = 1 << 21,
[Label ("non-monotouch")]
NonMonotouch = 1 << 22,
[Label ("old-simulator")]
OldiOSSimulator = 1 << 23,
[Label ("system-permission")]
SystemPermission = 1 << 24,
[Label ("tvos")]
tvOS = 1 << 15,
[Label ("watchos")]
watchOS = 1 << 26,
[Label ("xtro")]
Xtro = 1 << 27,
[Label ("all")]
All = 0xFFFFFFFF,
}
static class TestLabelExtensions {
public static string GetLabel (this TestLabel self)
{
var enumType = typeof(TestLabel);
var name = Enum.GetName (typeof (TestLabel), self);
var attr = enumType.GetField (name).GetCustomAttribute<LabelAttribute> ();
return attr.Label;
}
public static TestLabel GetLabel (this string self)
{
foreach (var obj in Enum.GetValues (typeof (TestLabel))) {
if (obj is TestLabel value && value.GetLabel () == self) {
return value;
}
}
throw new InvalidOperationException ($"Unknown label '{self}'");
}
}
}

Просмотреть файл

@ -161,6 +161,7 @@
<Compile Include="TestImporter\Xamarin\AssemblyLocator.cs" />
<Compile Include="TestImporter\Xamarin\ProjectFilter.cs" />
<Compile Include="TestImporter\Xamarin\TestAssemblyDefinition.cs" />
<Compile Include="TestLabel.cs" />
<Compile Include="TestPlatform.cs" />
<Compile Include="TestPlatformExtensions.cs" />
<Compile Include="TestProject.cs" />