From 2187eadf05f1ba6c798eb5603ad6c075107b9ac6 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 19 May 2020 11:13:31 -0400 Subject: [PATCH] [Harness] Move the generation of the variations out. (#8611) Try to make the Jenkins class do a single task. Move the generation of the variations logic out (which is hard to test atm, but will be doable in a second round). Moved some useful methos also out and add tests. --- tests/xharness/Jenkins/Jenkins.cs | 305 +----------------- tests/xharness/Jenkins/TestData.cs | 25 ++ .../xharness/Jenkins/TestTasks/RunTestTask.cs | 1 - .../xharness/Jenkins/TestVariationsFactory.cs | 231 +++++++++++++ tests/xharness/TestPlatformExtensions.cs | 54 ++++ tests/xharness/TestTargetExtensions.cs | 29 ++ .../Tests/TestPlatformExtensionsTests.cs | 93 ++++++ .../Tests/TestTargetExtensionsTests.cs | 28 ++ .../Xharness.Tests/Xharness.Tests.csproj | 2 + tests/xharness/xharness.csproj | 4 + 10 files changed, 479 insertions(+), 293 deletions(-) create mode 100644 tests/xharness/Jenkins/TestData.cs create mode 100644 tests/xharness/Jenkins/TestVariationsFactory.cs create mode 100644 tests/xharness/TestPlatformExtensions.cs create mode 100644 tests/xharness/TestTargetExtensions.cs create mode 100644 tests/xharness/Xharness.Tests/Tests/TestPlatformExtensionsTests.cs create mode 100644 tests/xharness/Xharness.Tests/Tests/TestTargetExtensionsTests.cs diff --git a/tests/xharness/Jenkins/Jenkins.cs b/tests/xharness/Jenkins/Jenkins.cs index 9e0d00ffb7..362132c8dd 100644 --- a/tests/xharness/Jenkins/Jenkins.cs +++ b/tests/xharness/Jenkins/Jenkins.cs @@ -17,12 +17,13 @@ using Microsoft.DotNet.XHarness.iOS.Shared.Listeners; namespace Xharness.Jenkins { public class Jenkins : IResourceManager { - readonly ISimulatorLoader simulators; + public readonly ISimulatorLoader Simulators; readonly IHardwareDeviceLoader devices; readonly IProcessManager processManager; readonly IResultParser resultParser; readonly ITunnelBore tunnelBore; readonly TestSelector testSelector; + readonly TestVariationsFactory testVariationsFactory; bool populating = true; @@ -109,9 +110,10 @@ namespace Xharness.Jenkins { this.resultParser = resultParser ?? throw new ArgumentNullException (nameof (resultParser)); this.tunnelBore = tunnelBore ?? throw new ArgumentNullException (nameof (tunnelBore)); Harness = harness ?? throw new ArgumentNullException (nameof (harness)); - simulators = new SimulatorLoader (processManager); + Simulators = new SimulatorLoader (processManager); devices = new HardwareDeviceLoader (processManager); testSelector = new TestSelector (this, processManager, new GitHub (harness, processManager)); + testVariationsFactory = new TestVariationsFactory (this, processManager); } Task LoadAsync (ref ILog log, IDeviceLoader deviceManager, string name) @@ -156,57 +158,16 @@ namespace Xharness.Jenkins { Task LoadSimulatorsAndDevicesAsync () { var devs = LoadAsync (ref DeviceLoadLog, devices, "Device"); - var sims = LoadAsync (ref SimulatorLoadLog, simulators, "Simulator"); + var sims = LoadAsync (ref SimulatorLoadLog, Simulators, "Simulator"); return Task.WhenAll (devs, sims); } - TestTarget[] GetAppRunnerTargets (TestPlatform platform) - { - switch (platform) { - case TestPlatform.tvOS: - return new TestTarget [] { TestTarget.Simulator_tvOS }; - case TestPlatform.watchOS: - case TestPlatform.watchOS_32: - case TestPlatform.watchOS_64_32: - return new TestTarget [] { TestTarget.Simulator_watchOS }; - case TestPlatform.iOS_Unified: - return new TestTarget [] { TestTarget.Simulator_iOS32, TestTarget.Simulator_iOS64 }; - case TestPlatform.iOS_Unified32: - return new TestTarget [] { TestTarget.Simulator_iOS32 }; - case TestPlatform.iOS_Unified64: - case TestPlatform.iOS_TodayExtension64: - return new TestTarget [] { TestTarget.Simulator_iOS64 }; - default: - throw new NotImplementedException (platform.ToString ()); - } - } - - string GetSimulatorMinVersion (TestPlatform platform) - { - switch (platform) { - case TestPlatform.iOS: - case TestPlatform.iOS_Unified: - case TestPlatform.iOS_TodayExtension64: - case TestPlatform.iOS_Unified32: - case TestPlatform.iOS_Unified64: - return "iOS " + SdkVersions.MiniOSSimulator; - case TestPlatform.tvOS: - return "tvOS " + SdkVersions.MinTVOSSimulator; - case TestPlatform.watchOS: - case TestPlatform.watchOS_32: - case TestPlatform.watchOS_64_32: - return "watchOS " + SdkVersions.MinWatchOSSimulator; - default: - throw new NotImplementedException (platform.ToString ()); - } - } - IEnumerable CreateRunSimulatorTaskAsync (MSBuildTask buildTask) { var runtasks = new List (); - TestTarget [] targets = GetAppRunnerTargets (buildTask.Platform); + TestTarget [] targets = buildTask.Platform.GetAppRunnerTargets (); TestPlatform [] platforms; bool [] ignored; @@ -233,10 +194,10 @@ namespace Xharness.Jenkins { } for (int i = 0; i < targets.Length; i++) { - var sims = simulators.SelectDevices (targets [i], SimulatorLoadLog, false); + var sims = Simulators.SelectDevices (targets [i], SimulatorLoadLog, false); runtasks.Add (new RunSimulatorTask ( jenkins: this, - simulators: simulators, + simulators: Simulators, buildTask: buildTask, processManager: processManager, tunnelBore: tunnelBore, @@ -274,246 +235,6 @@ namespace Xharness.Jenkins { return true; } - class TestData - { - public string Variation; - public string MTouchExtraArgs; - public string MonoBundlingExtraArgs; // mmp - public string KnownFailure; - public bool Debug; - public bool Profiling; - public string LinkMode; - public string Defines; - public string Undefines; - public bool? Ignored; - public bool EnableSGenConc; - public bool UseThumb; - public MonoNativeFlavor MonoNativeFlavor; - public MonoNativeLinkMode MonoNativeLinkMode; - public IEnumerable Candidates; - } - - IEnumerable GetTestData (RunTestTask test) - { - // This function returns additional test configurations (in addition to the default one) for the specific test - - MonoNativeFlavor flavor; - switch (test.TestName) { - case "mono-native-compat": - flavor = MonoNativeFlavor.Compat; - break; - case "mono-native-unified": - flavor = MonoNativeFlavor.Unified; - break; - default: - flavor = MonoNativeFlavor.None; - break; - } - - // 32-bit interpreter doesn't work yet: https://github.com/mono/mono/issues/9871 - var supports_interpreter = test.Platform != TestPlatform.iOS_Unified32; - var supports_dynamic_registrar_on_device = test.Platform == TestPlatform.iOS_Unified64 || test.Platform == TestPlatform.tvOS; - - switch (test.ProjectPlatform) { - case "iPhone": - // arm64_32 is only supported for Release builds for now. - // arm32 bits too big for debug builds - https://github.com/xamarin/maccore/issues/2080 - var supports_debug = test.Platform != TestPlatform.watchOS_64_32 && !(test.TestName == "dont link" && test.Platform == TestPlatform.iOS_Unified32); - - /* we don't add --assembly-build-target=@all=staticobject because that's the default in all our test projects */ - if (supports_debug) { - yield return new TestData { Variation = "AssemblyBuildTarget: dylib (debug)", MTouchExtraArgs = $"--assembly-build-target=@all=dynamiclibrary {test.TestProject.MTouchExtraArgs}", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Dynamic, MonoNativeFlavor = flavor }; - yield return new TestData { Variation = "AssemblyBuildTarget: SDK framework (debug)", MTouchExtraArgs = $"--assembly-build-target=@sdk=framework=Xamarin.Sdk --assembly-build-target=@all=staticobject {test.TestProject.MTouchExtraArgs}", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, MonoNativeFlavor = flavor }; - yield return new TestData { Variation = "AssemblyBuildTarget: dylib (debug, profiling)", MTouchExtraArgs = $"--assembly-build-target=@all=dynamiclibrary {test.TestProject.MTouchExtraArgs}", Debug = true, Profiling = true, MonoNativeLinkMode = MonoNativeLinkMode.Dynamic, MonoNativeFlavor = flavor }; - yield return new TestData { Variation = "AssemblyBuildTarget: SDK framework (debug, profiling)", MTouchExtraArgs = $"--assembly-build-target=@sdk=framework=Xamarin.Sdk --assembly-build-target=@all=staticobject {test.TestProject.MTouchExtraArgs}", Debug = true, Profiling = true, MonoNativeLinkMode = MonoNativeLinkMode.Static, MonoNativeFlavor = flavor }; - } - - if (test.ProjectConfiguration.Contains ("Debug")) - yield return new TestData { Variation = "Release", MTouchExtraArgs = test.TestProject.MTouchExtraArgs, Debug = false, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static }; - if (test.Platform == TestPlatform.iOS_Unified32) - yield return new TestData { Variation = "Release: UseThumb", MTouchExtraArgs = test.TestProject.MTouchExtraArgs, Debug = false, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, UseThumb = true }; - yield return new TestData { Variation = "AssemblyBuildTarget: SDK framework (release)", MTouchExtraArgs = $"--assembly-build-target=@sdk=framework=Xamarin.Sdk --assembly-build-target=@all=staticobject {test.TestProject.MTouchExtraArgs}", Debug = false, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, MonoNativeFlavor = flavor }; - - switch (test.TestName) { - case "monotouch-test": - if (supports_dynamic_registrar_on_device) - yield return new TestData { Variation = "Debug (dynamic registrar)", MTouchExtraArgs = "--registrar:dynamic", Debug = true, Profiling = false }; - yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, Defines = "OPTIMIZEALL" }; - if (supports_debug) { - yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = true, Profiling = false, Defines = "OPTIMIZEALL" }; - yield return new TestData { Variation = "Debug: SGenConc", MTouchExtraArgs = "", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, EnableSGenConc = true}; - } - if (supports_interpreter) { - if (supports_debug) { - yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME" }; - yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME" }; - } - yield return new TestData { Variation = "Release (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = false, Profiling = false, Undefines = "FULL_AOT_RUNTIME" }; - } - break; - case string name when name.StartsWith ("mscorlib", StringComparison.Ordinal): - if (supports_debug) - yield return new TestData { Variation = "Debug: SGenConc", MTouchExtraArgs = "", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, EnableSGenConc = true}; - if (supports_interpreter) { - if (supports_debug) { - yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = "#1683" }; - yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = "#1682" }; - } - yield return new TestData { Variation = "Release (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = false, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = "#1682" }; - } - break; - } - break; - case "iPhoneSimulator": - switch (test.TestName) { - case "monotouch-test": - // The default is to run monotouch-test with the dynamic registrar (in the simulator), so that's already covered - yield return new TestData { Variation = "Debug (LinkSdk)", Debug = true, Profiling = false, LinkMode = "LinkSdk" }; - yield return new TestData { Variation = "Debug (static registrar)", MTouchExtraArgs = "--registrar:static", Debug = true, Profiling = false, Undefines = "DYNAMIC_REGISTRAR" }; - yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR" }; - yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = !IncludeAll }; - break; - case "introspection": - foreach (var target in GetAppRunnerTargets (test.Platform)) - yield return new TestData { - Variation = $"Debug ({GetSimulatorMinVersion (test.Platform)})", - Debug = true, - Candidates = simulators.SelectDevices (target, SimulatorLoadLog, true), - Ignored = !IncludeOldSimulatorTests, - }; - break; - } - break; - case "AnyCPU": - case "x86": - switch (test.TestName) { - case "xammac tests": - switch (test.ProjectConfiguration) { - case "Release": - yield return new TestData { Variation = "Release (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all", Debug = false, LinkMode = "Full", Defines = "OPTIMIZEALL"}; - break; - case "Debug": - yield return new TestData { Variation = "Debug (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, LinkMode = "Full", Defines = "OPTIMIZEALL", Ignored = !IncludeAll }; - break; - } - break; - } - break; - default: - throw new NotImplementedException (test.ProjectPlatform); - } - } - - IEnumerable CreateTestVariations (IEnumerable tests, Func, T> creator) where T: RunTestTask - { - foreach (var task in tests) { - if (string.IsNullOrEmpty (task.Variation)) - task.Variation = task.ProjectConfiguration.Contains ("Debug") ? "Debug" : "Release"; - } - - var rv = new List (tests); - foreach (var task in tests.ToArray ()) { - foreach (var test_data in GetTestData (task)) { - var variation = test_data.Variation; - var mtouch_extra_args = test_data.MTouchExtraArgs; - var bundling_extra_args = test_data.MonoBundlingExtraArgs; - var configuration = test_data.Debug ? task.ProjectConfiguration : task.ProjectConfiguration.Replace ("Debug", "Release"); - var debug = test_data.Debug; - var profiling = test_data.Profiling; - var link_mode = test_data.LinkMode; - var defines = test_data.Defines; - var undefines = test_data.Undefines; - var ignored = test_data.Ignored; - var known_failure = test_data.KnownFailure; - var candidates = test_data.Candidates; - - if (!string.IsNullOrEmpty (known_failure)) - ignored = true; - - var clone = task.TestProject.Clone (); - var clone_task = Task.Run (async () => { - await task.BuildTask.InitialTask; // this is the project cloning above - await clone.CreateCopyAsync (task); - - var isMac = false; - var canSymlink = false; - switch (task.Platform) { - case TestPlatform.Mac: - case TestPlatform.Mac_Modern: - case TestPlatform.Mac_Full: - case TestPlatform.Mac_System: - isMac = true; - break; - case TestPlatform.iOS: - case TestPlatform.iOS_TodayExtension64: - case TestPlatform.iOS_Unified: - case TestPlatform.iOS_Unified32: - case TestPlatform.iOS_Unified64: - canSymlink = true; - break; - } - - if (!string.IsNullOrEmpty (mtouch_extra_args)) - clone.Xml.AddExtraMtouchArgs (mtouch_extra_args, task.ProjectPlatform, configuration); - if (!string.IsNullOrEmpty (bundling_extra_args)) - clone.Xml.AddMonoBundlingExtraArgs (bundling_extra_args, task.ProjectPlatform, configuration); - if (!string.IsNullOrEmpty (link_mode)) - clone.Xml.SetNode (isMac ? "LinkMode" : "MtouchLink", link_mode, task.ProjectPlatform, configuration); - if (!string.IsNullOrEmpty (defines)) { - clone.Xml.AddAdditionalDefines (defines, task.ProjectPlatform, configuration); - if (clone.ProjectReferences != null) { - foreach (var pr in clone.ProjectReferences) { - pr.Xml.AddAdditionalDefines (defines, task.ProjectPlatform, configuration); - pr.Xml.Save (pr.Path); - } - } - } - if (!string.IsNullOrEmpty (undefines)) { - clone.Xml.RemoveDefines (undefines, task.ProjectPlatform, configuration); - if (clone.ProjectReferences != null) { - foreach (var pr in clone.ProjectReferences) { - pr.Xml.RemoveDefines (undefines, task.ProjectPlatform, configuration); - pr.Xml.Save (pr.Path); - } - } - } - clone.Xml.SetNode (isMac ? "Profiling" : "MTouchProfiling", profiling ? "True" : "False", task.ProjectPlatform, configuration); - if (test_data.MonoNativeFlavor != MonoNativeFlavor.None) { - var mono_native_link = test_data.MonoNativeLinkMode; - if (!canSymlink && mono_native_link == MonoNativeLinkMode.Symlink) - mono_native_link = MonoNativeLinkMode.Static; - MonoNativeHelper.AddProjectDefines (clone.Xml, test_data.MonoNativeFlavor, mono_native_link, task.ProjectPlatform, configuration); - } - if (test_data.EnableSGenConc) - clone.Xml.SetNode ("MtouchEnableSGenConc", "true", task.ProjectPlatform, configuration); - if (test_data.UseThumb) // no need to check the platform, already done at the data iterator - clone.Xml.SetNode ("MtouchUseThumb", "true", task.ProjectPlatform, configuration); - - if (!debug && !isMac) - clone.Xml.SetMtouchUseLlvm (true, task.ProjectPlatform, configuration); - clone.Xml.Save (clone.Path); - }); - - var build = new MSBuildTask (jenkins: this, testProject: clone, processManager: processManager) { - ProjectConfiguration = configuration, - ProjectPlatform = task.ProjectPlatform, - Platform = task.Platform, - InitialTask = clone_task, - TestName = clone.Name, - }; - T newVariation = creator (build, task, candidates); - newVariation.Variation = variation; - newVariation.Ignored = ignored ?? task.Ignored; - newVariation.BuildOnly = task.BuildOnly; - newVariation.TimeoutMultiplier = task.TimeoutMultiplier; - newVariation.KnownFailure = known_failure; - rv.Add (newVariation); - } - } - - return rv; - } - async Task> CreateRunSimulatorTasksAsync () { var runSimulatorTasks = new List (); @@ -561,10 +282,10 @@ namespace Xharness.Jenkins { } } - var testVariations = CreateTestVariations (runSimulatorTasks, (buildTask, test, candidates) => + var testVariations = testVariationsFactory.CreateTestVariations (runSimulatorTasks, (buildTask, test, candidates) => new RunSimulatorTask ( jenkins: this, - simulators: simulators, + simulators: Simulators, buildTask: buildTask, processManager: processManager, tunnelBore: tunnelBore, @@ -714,7 +435,7 @@ namespace Xharness.Jenkins { rv.AddRange (projectTasks); } - return Task.FromResult> (CreateTestVariations (rv, (buildTask, test, candidates) + return Task.FromResult> (testVariationsFactory.CreateTestVariations (rv, (buildTask, test, candidates) => new RunDeviceTask ( jenkins: this, devices: devices, @@ -851,7 +572,7 @@ namespace Xharness.Jenkins { TestName = project.Name, IsUnitTest = true, }; - execs = CreateTestVariations (new [] { exec }, (buildTask, test, candidates) => + execs = testVariationsFactory.CreateTestVariations (new [] { exec }, (buildTask, test, candidates) => new MacExecuteTask (this, buildTask, processManager, crashReportSnapshotFactory) { IsUnitTest = true } ); } @@ -1368,7 +1089,7 @@ namespace Xharness.Jenkins { LoadAsync (ref DeviceLoadLog, devices, "Device").DoNotAwait (); break; case "/reload-simulators": - LoadAsync (ref SimulatorLoadLog, simulators, "Simulator").DoNotAwait (); + LoadAsync (ref SimulatorLoadLog, Simulators, "Simulator").DoNotAwait (); break; case "/quit": using (var writer = new StreamWriter (response.OutputStream)) { diff --git a/tests/xharness/Jenkins/TestData.cs b/tests/xharness/Jenkins/TestData.cs new file mode 100644 index 0000000000..d15212e996 --- /dev/null +++ b/tests/xharness/Jenkins/TestData.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using Microsoft.DotNet.XHarness.iOS.Shared; +using Microsoft.DotNet.XHarness.iOS.Shared.Hardware; + +namespace Xharness.Jenkins { + + class TestData + { + public string Variation; + public string MTouchExtraArgs; + public string MonoBundlingExtraArgs; // mmp + public string KnownFailure; + public bool Debug; + public bool Profiling; + public string LinkMode; + public string Defines; + public string Undefines; + public bool? Ignored; + public bool EnableSGenConc; + public bool UseThumb; + public MonoNativeFlavor MonoNativeFlavor; + public MonoNativeLinkMode MonoNativeLinkMode; + public IEnumerable Candidates; + } +} diff --git a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs index ce4bd7544c..8079e17024 100644 --- a/tests/xharness/Jenkins/TestTasks/RunTestTask.cs +++ b/tests/xharness/Jenkins/TestTasks/RunTestTask.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.DotNet.XHarness.iOS.Shared; using Microsoft.DotNet.XHarness.iOS.Shared.Execution; -using Microsoft.DotNet.XHarness.iOS.Shared.Listeners; using Microsoft.DotNet.XHarness.iOS.Shared.Logging; using Xharness.TestTasks; diff --git a/tests/xharness/Jenkins/TestVariationsFactory.cs b/tests/xharness/Jenkins/TestVariationsFactory.cs new file mode 100644 index 0000000000..8a77439569 --- /dev/null +++ b/tests/xharness/Jenkins/TestVariationsFactory.cs @@ -0,0 +1,231 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.DotNet.XHarness.iOS.Shared; +using Microsoft.DotNet.XHarness.iOS.Shared.Execution; +using Microsoft.DotNet.XHarness.iOS.Shared.Hardware; +using Microsoft.DotNet.XHarness.iOS.Shared.Utilities; +using Xharness.Jenkins.TestTasks; + +namespace Xharness.Jenkins { + + class TestVariationsFactory { + + readonly Jenkins jenkins; + readonly IProcessManager processManager; + + public TestVariationsFactory (Jenkins jenkins, IProcessManager processManager) + { + this.jenkins = jenkins ?? throw new ArgumentNullException (nameof (jenkins)); + this.processManager = processManager ?? throw new ArgumentNullException (nameof (processManager)); + } + + IEnumerable GetTestData (RunTestTask test) + { + // This function returns additional test configurations (in addition to the default one) for the specific test + + MonoNativeFlavor flavor; + switch (test.TestName) { + case "mono-native-compat": + flavor = MonoNativeFlavor.Compat; + break; + case "mono-native-unified": + flavor = MonoNativeFlavor.Unified; + break; + default: + flavor = MonoNativeFlavor.None; + break; + } + + // 32-bit interpreter doesn't work yet: https://github.com/mono/mono/issues/9871 + var supports_interpreter = test.Platform != TestPlatform.iOS_Unified32; + var supports_dynamic_registrar_on_device = test.Platform == TestPlatform.iOS_Unified64 || test.Platform == TestPlatform.tvOS; + + switch (test.ProjectPlatform) { + case "iPhone": + // arm64_32 is only supported for Release builds for now. + // arm32 bits too big for debug builds - https://github.com/xamarin/maccore/issues/2080 + var supports_debug = test.Platform != TestPlatform.watchOS_64_32 && !(test.TestName == "dont link" && test.Platform == TestPlatform.iOS_Unified32); + + /* we don't add --assembly-build-target=@all=staticobject because that's the default in all our test projects */ + if (supports_debug) { + yield return new TestData { Variation = "AssemblyBuildTarget: dylib (debug)", MTouchExtraArgs = $"--assembly-build-target=@all=dynamiclibrary {test.TestProject.MTouchExtraArgs}", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Dynamic, MonoNativeFlavor = flavor }; + yield return new TestData { Variation = "AssemblyBuildTarget: SDK framework (debug)", MTouchExtraArgs = $"--assembly-build-target=@sdk=framework=Xamarin.Sdk --assembly-build-target=@all=staticobject {test.TestProject.MTouchExtraArgs}", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, MonoNativeFlavor = flavor }; + yield return new TestData { Variation = "AssemblyBuildTarget: dylib (debug, profiling)", MTouchExtraArgs = $"--assembly-build-target=@all=dynamiclibrary {test.TestProject.MTouchExtraArgs}", Debug = true, Profiling = true, MonoNativeLinkMode = MonoNativeLinkMode.Dynamic, MonoNativeFlavor = flavor }; + yield return new TestData { Variation = "AssemblyBuildTarget: SDK framework (debug, profiling)", MTouchExtraArgs = $"--assembly-build-target=@sdk=framework=Xamarin.Sdk --assembly-build-target=@all=staticobject {test.TestProject.MTouchExtraArgs}", Debug = true, Profiling = true, MonoNativeLinkMode = MonoNativeLinkMode.Static, MonoNativeFlavor = flavor }; + } + + if (test.ProjectConfiguration.Contains ("Debug")) + yield return new TestData { Variation = "Release", MTouchExtraArgs = test.TestProject.MTouchExtraArgs, Debug = false, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static }; + if (test.Platform == TestPlatform.iOS_Unified32) + yield return new TestData { Variation = "Release: UseThumb", MTouchExtraArgs = test.TestProject.MTouchExtraArgs, Debug = false, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, UseThumb = true }; + yield return new TestData { Variation = "AssemblyBuildTarget: SDK framework (release)", MTouchExtraArgs = $"--assembly-build-target=@sdk=framework=Xamarin.Sdk --assembly-build-target=@all=staticobject {test.TestProject.MTouchExtraArgs}", Debug = false, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, MonoNativeFlavor = flavor }; + + switch (test.TestName) { + case "monotouch-test": + if (supports_dynamic_registrar_on_device) + yield return new TestData { Variation = "Debug (dynamic registrar)", MTouchExtraArgs = "--registrar:dynamic", Debug = true, Profiling = false }; + yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, Defines = "OPTIMIZEALL" }; + if (supports_debug) { + yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = true, Profiling = false, Defines = "OPTIMIZEALL" }; + yield return new TestData { Variation = "Debug: SGenConc", MTouchExtraArgs = "", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, EnableSGenConc = true}; + } + if (supports_interpreter) { + if (supports_debug) { + yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME" }; + yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME" }; + } + yield return new TestData { Variation = "Release (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = false, Profiling = false, Undefines = "FULL_AOT_RUNTIME" }; + } + break; + case string name when name.StartsWith ("mscorlib", StringComparison.Ordinal): + if (supports_debug) + yield return new TestData { Variation = "Debug: SGenConc", MTouchExtraArgs = "", Debug = true, Profiling = false, MonoNativeLinkMode = MonoNativeLinkMode.Static, EnableSGenConc = true}; + if (supports_interpreter) { + if (supports_debug) { + yield return new TestData { Variation = "Debug (interpreter)", MTouchExtraArgs = "--interpreter", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = "#1683" }; + yield return new TestData { Variation = "Debug (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = true, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = "#1682" }; + } + yield return new TestData { Variation = "Release (interpreter -mscorlib)", MTouchExtraArgs = "--interpreter=-mscorlib", Debug = false, Profiling = false, Undefines = "FULL_AOT_RUNTIME", KnownFailure = "#1682" }; + } + break; + } + break; + case "iPhoneSimulator": + switch (test.TestName) { + case "monotouch-test": + // The default is to run monotouch-test with the dynamic registrar (in the simulator), so that's already covered + yield return new TestData { Variation = "Debug (LinkSdk)", Debug = true, Profiling = false, LinkMode = "LinkSdk" }; + yield return new TestData { Variation = "Debug (static registrar)", MTouchExtraArgs = "--registrar:static", Debug = true, Profiling = false, Undefines = "DYNAMIC_REGISTRAR" }; + yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR" }; + yield return new TestData { Variation = "Debug (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, Profiling = false, LinkMode = "Full", Defines = "OPTIMIZEALL", Undefines = "DYNAMIC_REGISTRAR", Ignored = !jenkins.IncludeAll }; + break; + case "introspection": + foreach (var target in test.Platform.GetAppRunnerTargets ()) + yield return new TestData { + Variation = $"Debug ({test.Platform.GetSimulatorMinVersion ()})", + Debug = true, + Candidates = jenkins.Simulators.SelectDevices (target, jenkins.SimulatorLoadLog, true), + Ignored = !jenkins.IncludeOldSimulatorTests, + }; + break; + } + break; + case "AnyCPU": + case "x86": + switch (test.TestName) { + case "xammac tests": + switch (test.ProjectConfiguration) { + case "Release": + yield return new TestData { Variation = "Release (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all", Debug = false, LinkMode = "Full", Defines = "OPTIMIZEALL"}; + break; + case "Debug": + yield return new TestData { Variation = "Debug (all optimizations)", MonoBundlingExtraArgs = "--registrar:static --optimize:all,-remove-uithread-checks", Debug = true, LinkMode = "Full", Defines = "OPTIMIZEALL", Ignored = !jenkins.IncludeAll }; + break; + } + break; + } + break; + default: + throw new NotImplementedException (test.ProjectPlatform); + } + } + + internal IEnumerable CreateTestVariations (IEnumerable tests, Func, T> creator) where T : RunTestTask + { + foreach (var task in tests) { + if (string.IsNullOrEmpty (task.Variation)) + task.Variation = task.ProjectConfiguration.Contains ("Debug") ? "Debug" : "Release"; + } + + var rv = new List (tests); + foreach (var task in tests.ToArray ()) { + foreach (var test_data in GetTestData (task)) { + var variation = test_data.Variation; + var mtouch_extra_args = test_data.MTouchExtraArgs; + var bundling_extra_args = test_data.MonoBundlingExtraArgs; + var configuration = test_data.Debug ? task.ProjectConfiguration : task.ProjectConfiguration.Replace ("Debug", "Release"); + var debug = test_data.Debug; + var profiling = test_data.Profiling; + var link_mode = test_data.LinkMode; + var defines = test_data.Defines; + var undefines = test_data.Undefines; + var ignored = test_data.Ignored; + var known_failure = test_data.KnownFailure; + var candidates = test_data.Candidates; + + if (!string.IsNullOrEmpty (known_failure)) + ignored = true; + + var clone = task.TestProject.Clone (); + var clone_task = Task.Run (async () => { + await task.BuildTask.InitialTask; // this is the project cloning above + await clone.CreateCopyAsync (task); + + var isMac = task.Platform.IsMac (); + var canSymlink = task.Platform.CanSymlink(); + + if (!string.IsNullOrEmpty (mtouch_extra_args)) + clone.Xml.AddExtraMtouchArgs (mtouch_extra_args, task.ProjectPlatform, configuration); + if (!string.IsNullOrEmpty (bundling_extra_args)) + clone.Xml.AddMonoBundlingExtraArgs (bundling_extra_args, task.ProjectPlatform, configuration); + if (!string.IsNullOrEmpty (link_mode)) + clone.Xml.SetNode (isMac ? "LinkMode" : "MtouchLink", link_mode, task.ProjectPlatform, configuration); + if (!string.IsNullOrEmpty (defines)) { + clone.Xml.AddAdditionalDefines (defines, task.ProjectPlatform, configuration); + if (clone.ProjectReferences != null) { + foreach (var pr in clone.ProjectReferences) { + pr.Xml.AddAdditionalDefines (defines, task.ProjectPlatform, configuration); + pr.Xml.Save (pr.Path); + } + } + } + if (!string.IsNullOrEmpty (undefines)) { + clone.Xml.RemoveDefines (undefines, task.ProjectPlatform, configuration); + if (clone.ProjectReferences != null) { + foreach (var pr in clone.ProjectReferences) { + pr.Xml.RemoveDefines (undefines, task.ProjectPlatform, configuration); + pr.Xml.Save (pr.Path); + } + } + } + clone.Xml.SetNode (isMac ? "Profiling" : "MTouchProfiling", profiling ? "True" : "False", task.ProjectPlatform, configuration); + if (test_data.MonoNativeFlavor != MonoNativeFlavor.None) { + var mono_native_link = test_data.MonoNativeLinkMode; + if (!canSymlink && mono_native_link == MonoNativeLinkMode.Symlink) + mono_native_link = MonoNativeLinkMode.Static; + MonoNativeHelper.AddProjectDefines (clone.Xml, test_data.MonoNativeFlavor, mono_native_link, task.ProjectPlatform, configuration); + } + if (test_data.EnableSGenConc) + clone.Xml.SetNode ("MtouchEnableSGenConc", "true", task.ProjectPlatform, configuration); + if (test_data.UseThumb) // no need to check the platform, already done at the data iterator + clone.Xml.SetNode ("MtouchUseThumb", "true", task.ProjectPlatform, configuration); + + if (!debug && !isMac) + clone.Xml.SetMtouchUseLlvm (true, task.ProjectPlatform, configuration); + clone.Xml.Save (clone.Path); + }); + + var build = new MSBuildTask (jenkins: jenkins, testProject: clone, processManager: processManager) { + ProjectConfiguration = configuration, + ProjectPlatform = task.ProjectPlatform, + Platform = task.Platform, + InitialTask = clone_task, + TestName = clone.Name, + }; + T newVariation = creator (build, task, candidates); + newVariation.Variation = variation; + newVariation.Ignored = ignored ?? task.Ignored; + newVariation.BuildOnly = task.BuildOnly; + newVariation.TimeoutMultiplier = task.TimeoutMultiplier; + newVariation.KnownFailure = known_failure; + rv.Add (newVariation); + } + } + + return rv; + } + + } +} diff --git a/tests/xharness/TestPlatformExtensions.cs b/tests/xharness/TestPlatformExtensions.cs new file mode 100644 index 0000000000..285a1bb606 --- /dev/null +++ b/tests/xharness/TestPlatformExtensions.cs @@ -0,0 +1,54 @@ +using System; +using Microsoft.DotNet.XHarness.iOS.Shared; + +namespace Xharness { + public static class TestPlatformExtensions { + + public static string GetSimulatorMinVersion (this TestPlatform platform) + { + switch (platform) { + case TestPlatform.iOS: + case TestPlatform.iOS_Unified: + case TestPlatform.iOS_TodayExtension64: + case TestPlatform.iOS_Unified32: + case TestPlatform.iOS_Unified64: + return "iOS " + SdkVersions.MiniOSSimulator; + case TestPlatform.tvOS: + return "tvOS " + SdkVersions.MinTVOSSimulator; + case TestPlatform.watchOS: + case TestPlatform.watchOS_32: + case TestPlatform.watchOS_64_32: + return "watchOS " + SdkVersions.MinWatchOSSimulator; + default: + throw new NotImplementedException (platform.ToString ()); + } + } + + public static bool IsMac (this TestPlatform platform) + { + switch (platform) { + case TestPlatform.Mac: + case TestPlatform.Mac_Modern: + case TestPlatform.Mac_Full: + case TestPlatform.Mac_System: + return true; + default: + return false; + } + } + + public static bool CanSymlink (this TestPlatform platform) + { + switch (platform) { + case TestPlatform.iOS: + case TestPlatform.iOS_TodayExtension64: + case TestPlatform.iOS_Unified: + case TestPlatform.iOS_Unified32: + case TestPlatform.iOS_Unified64: + return true; + default: + return false; + } + } + } +} diff --git a/tests/xharness/TestTargetExtensions.cs b/tests/xharness/TestTargetExtensions.cs new file mode 100644 index 0000000000..5451c6a24b --- /dev/null +++ b/tests/xharness/TestTargetExtensions.cs @@ -0,0 +1,29 @@ +using System; +using Microsoft.DotNet.XHarness.iOS.Shared; + +namespace Xharness { + + public static class TestTargetExtensions { + + public static TestTarget[] GetAppRunnerTargets (this TestPlatform platform) + { + switch (platform) { + case TestPlatform.tvOS: + return new TestTarget [] { TestTarget.Simulator_tvOS }; + case TestPlatform.watchOS: + case TestPlatform.watchOS_32: + case TestPlatform.watchOS_64_32: + return new TestTarget [] { TestTarget.Simulator_watchOS }; + case TestPlatform.iOS_Unified: + return new TestTarget [] { TestTarget.Simulator_iOS32, TestTarget.Simulator_iOS64 }; + case TestPlatform.iOS_Unified32: + return new TestTarget [] { TestTarget.Simulator_iOS32 }; + case TestPlatform.iOS_Unified64: + case TestPlatform.iOS_TodayExtension64: + return new TestTarget [] { TestTarget.Simulator_iOS64 }; + default: + throw new NotImplementedException (platform.ToString ()); + } + } + } +} diff --git a/tests/xharness/Xharness.Tests/Tests/TestPlatformExtensionsTests.cs b/tests/xharness/Xharness.Tests/Tests/TestPlatformExtensionsTests.cs new file mode 100644 index 0000000000..a004dd33ec --- /dev/null +++ b/tests/xharness/Xharness.Tests/Tests/TestPlatformExtensionsTests.cs @@ -0,0 +1,93 @@ +using System.Collections; +using Microsoft.DotNet.XHarness.iOS.Shared; +using NUnit.Framework; + +namespace Xharness.Tests.Tests { + + [TestFixture] + public class TestPlatformExtensionsTests { + + public class TestCasesData { + public static IEnumerable GetSimulatorTestCases { + get { + foreach (var platform in new [] { TestPlatform.iOS, + TestPlatform.iOS_Unified, + TestPlatform.iOS_TodayExtension64, + TestPlatform.iOS_Unified32, + TestPlatform.iOS_Unified64,}) { + yield return new TestCaseData (platform).Returns ("iOS " + SdkVersions.MiniOSSimulator); + } + yield return new TestCaseData (TestPlatform.tvOS).Returns ("tvOS " + SdkVersions.MinTVOSSimulator); + foreach (var platform in new [] { TestPlatform.watchOS, + TestPlatform.watchOS_32, + TestPlatform.watchOS_64_32}) { + yield return new TestCaseData (platform).Returns ("watchOS " + SdkVersions.MinWatchOSSimulator); + } + } + } + + public static IEnumerable IsMacTestCases { + get { + + foreach (var platform in new [] { TestPlatform.None, + TestPlatform.All, + TestPlatform.iOS, + TestPlatform.iOS_Unified, + TestPlatform.iOS_Unified32, + TestPlatform.iOS_Unified64, + TestPlatform.iOS_TodayExtension64, + TestPlatform.tvOS, + TestPlatform.watchOS, + TestPlatform.watchOS_32, + TestPlatform.watchOS_64_32 }) { + yield return new TestCaseData (platform).Returns (false); + } + + foreach (var platform in new [] { TestPlatform.Mac, + TestPlatform.Mac_Modern, + TestPlatform.Mac_Full, + TestPlatform.Mac_System }) { + yield return new TestCaseData (platform).Returns (true); + } + } + } + + public static IEnumerable CanSymlinkTestCases { + get { + foreach (var platform in new [] { TestPlatform.iOS, + TestPlatform.iOS_TodayExtension64, + TestPlatform.iOS_Unified, + TestPlatform.iOS_Unified32, + TestPlatform.iOS_Unified64 }) { + yield return new TestCaseData (platform).Returns (true); + } + + foreach (var platform in new [] {TestPlatform.None, + + TestPlatform.tvOS, + TestPlatform.watchOS, + TestPlatform.watchOS_32, + TestPlatform.watchOS_64_32, + TestPlatform.Mac, + TestPlatform.Mac_Modern, + TestPlatform.Mac_Full, + TestPlatform.Mac_System}) { + yield return new TestCaseData (platform).Returns (false); + } + } + } + } + + [Test, TestCaseSource (typeof (TestCasesData), "GetSimulatorTestCases")] + public string GetSimulatorMinVersionTest (TestPlatform platform) + => platform.GetSimulatorMinVersion (); + + [Test, TestCaseSource (typeof (TestCasesData), "IsMacTestCases")] + public bool IsMacTest (TestPlatform platform) => platform.IsMac (); + + [Test, TestCaseSource (typeof (TestCasesData), "CanSymlinkTestCases")] + public bool CanSymlinkTest (TestPlatform platform) => platform.CanSymlink (); + + } + +} diff --git a/tests/xharness/Xharness.Tests/Tests/TestTargetExtensionsTests.cs b/tests/xharness/Xharness.Tests/Tests/TestTargetExtensionsTests.cs new file mode 100644 index 0000000000..d922e2ffef --- /dev/null +++ b/tests/xharness/Xharness.Tests/Tests/TestTargetExtensionsTests.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections; +using Microsoft.DotNet.XHarness.iOS.Shared; +using NUnit.Framework; + +namespace Xharness.Tests.Tests { + + [TestFixture] + public class TestTargetExtensionsTests { + + [TestCase (TestPlatform.tvOS, new [] { TestTarget.Simulator_tvOS })] + [TestCase (TestPlatform.watchOS, new [] { TestTarget.Simulator_watchOS })] + [TestCase (TestPlatform.watchOS_32, new [] { TestTarget.Simulator_watchOS })] + [TestCase (TestPlatform.watchOS_64_32, new [] { TestTarget.Simulator_watchOS })] + [TestCase (TestPlatform.iOS_Unified, new [] { TestTarget.Simulator_iOS32, TestTarget.Simulator_iOS64 })] + [TestCase (TestPlatform.iOS_Unified32, new [] { TestTarget.Simulator_iOS32 })] + [TestCase (TestPlatform.iOS_Unified64, new [] { TestTarget.Simulator_iOS64 })] + [TestCase (TestPlatform.iOS_TodayExtension64, new [] { TestTarget.Simulator_iOS64 })] + public void GetAppRunnerTargetsTest (TestPlatform platform, TestTarget[] expectedTargets) + { + var targets = platform.GetAppRunnerTargets (); + Assert.AreEqual (expectedTargets.Length, targets.Length); + foreach (var t in expectedTargets) { + Assert.Contains (t, targets); + } + } + } +} diff --git a/tests/xharness/Xharness.Tests/Xharness.Tests.csproj b/tests/xharness/Xharness.Tests/Xharness.Tests.csproj index d6c88d79af..bb5dfb2d77 100644 --- a/tests/xharness/Xharness.Tests/Xharness.Tests.csproj +++ b/tests/xharness/Xharness.Tests/Xharness.Tests.csproj @@ -61,6 +61,8 @@ + + diff --git a/tests/xharness/xharness.csproj b/tests/xharness/xharness.csproj index d8841185b9..be9a3dc28c 100644 --- a/tests/xharness/xharness.csproj +++ b/tests/xharness/xharness.csproj @@ -125,6 +125,10 @@ + + + +