[xharness] Add support for Mac Catalyst test suites.

tests/xharness/Harness.cs

@@ -557,6 +557,16 @@ namespace Xharness {
 						today_targets.Add (today);
 					}
 				}
+
+				if (!proj.SkipMacCatalystVariation) {
+					var macCatalyst = new MacCatalystTarget {
+						TemplateProjectPath = file,
+						Harness = this,
+						TestProject = proj,
+						ShouldSkipProjectGeneration = proj.IsDotNetProject,
+					};
+					macCatalyst.Execute ();
+				}
 			}
 
 			SolutionGenerator.CreateSolution (this, watchos_targets, "watchos", DevicePlatform.watchOS);

tests/xharness/Jenkins/Jenkins.cs

@@ -61,6 +61,7 @@ namespace Xharness.Jenkins {
 		public bool IncludeNonMonotouch = true;
 		public bool IncludeMonotouch = true;
 		public bool IncludeDotNet;
+		public bool IncludeMacCatalyst;
 
 		public bool CleanSuccessfulTestRuns = true;
 		public bool UninstallTestApp = true;
@@ -234,9 +235,65 @@ namespace Xharness.Jenkins {
 				Console.WriteLine ("Got device tasks completed");
 				Tasks.AddRange (v.Result);
 			});
+
+			// Generate Mac Catalyst tests
+			Tasks.AddRange (CreateMacCatalystTests (crashReportSnapshotFactory));
+
 			return Task.WhenAll (loadsim, loaddev);
 		}
 
+		IEnumerable<ITestTask> CreateMacCatalystTests (CrashSnapshotReporterFactory crashSnapshotReporterFactory)
+		{
+			var projectTasks = new List<RunTestTask> ();
+
+			foreach (var project in Harness.IOSTestProjects) {
+				if (!project.IsExecutableProject)
+					continue;
+
+				if (project.SkipMacCatalystVariation)
+					continue;
+
+				if (!project.GenerateVariations)
+					continue;
+
+				var ignored = project.Ignore ?? !IncludeMacCatalyst;
+				if (!IsIncluded (project))
+					ignored = true;
+
+				var macCatalystProject = project.GenerateVariations ? project.AsMacCatalystProject () : project;
+				var build = new MSBuildTask (jenkins: this, testProject: macCatalystProject, processManager: processManager) {
+					ProjectConfiguration = "Debug",
+					ProjectPlatform = "iPhoneSimulator",
+					Platform = TestPlatform.MacCatalyst,
+					TestName = project.Name,
+				};
+				build.CloneTestProject (MainLog, processManager, macCatalystProject, HarnessConfiguration.RootDirectory);
+
+				RunTestTask task;
+				if (project.IsNUnitProject) {
+					var dll = Path.Combine (Path.GetDirectoryName (build.TestProject.Path), project.Xml.GetOutputAssemblyPath (build.ProjectPlatform, build.ProjectConfiguration).Replace ('\\', '/'));
+					task = new NUnitExecuteTask (this, build, processManager) {
+						TestLibrary = dll,
+						Mode = "MacCatalyst",
+					};
+				} else {
+					task = new MacExecuteTask (this, build, processManager, crashSnapshotReporterFactory) {
+						IsUnitTest = true,
+					};
+				}
+				task.Ignored = ignored;
+				task.Platform = build.Platform;
+				task.TestName = project.Name;
+				task.Timeout = TimeSpan.FromMinutes (120);
+				task.Variation = task.ProjectConfiguration;
+				if (project.IsDotNetProject)
+					task.Variation += " [dotnet]";
+				projectTasks.Add (task);
+			}
+
+			return projectTasks;
+		}
+
 		public int Run ()
 		{
 			try {

tests/xharness/Jenkins/TestTasks/AppleTestTask.cs

@@ -44,6 +44,7 @@ namespace Xharness.Jenkins.TestTasks {
 			case TestPlatform.watchOS:
 			case TestPlatform.watchOS_32:
 			case TestPlatform.watchOS_64_32:
+			case TestPlatform.MacCatalyst:
 				process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot;
 				process.StartInfo.EnvironmentVariables ["MD_MTOUCH_SDK_ROOT"] = Path.Combine (Jenkins.Harness.IOS_DESTDIR, "Library", "Frameworks", "Xamarin.iOS.framework", "Versions", "Current");
 				process.StartInfo.EnvironmentVariables ["TargetFrameworkFallbackSearchPaths"] = Path.Combine (Jenkins.Harness.IOS_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks");
@@ -71,7 +72,7 @@ namespace Xharness.Jenkins.TestTasks {
 				process.StartInfo.EnvironmentVariables ["XAMMAC_FRAMEWORK_PATH"] = Path.Combine (Jenkins.Harness.MAC_DESTDIR, "Library", "Frameworks", "Xamarin.Mac.framework", "Versions", "Current");
 				break;
 			default:
-				throw new NotImplementedException ();
+				throw new NotImplementedException ($"Unknown test platform: {Platform}");
 			}
 
 			foreach (var kvp in Environment) {

tests/xharness/Jenkins/TestTasks/BuildProject.cs

@@ -32,6 +32,8 @@ namespace Xharness.Jenkins.TestTasks {
 			get {
 				if (TestProject.IsDotNetProject)
 					return false;
+				if (Platform == TestPlatform.MacCatalyst)
+					return false;
 				return TestProject.RestoreNugetsInProject || !string.IsNullOrEmpty (SolutionPath);
 			}
 		}

tests/xharness/Jenkins/TestTasks/MSBuild.cs

@@ -29,6 +29,8 @@ namespace Xharness.Jenkins.TestTasks {
 				args.Add ($"/p:Platform={projectPlatform}");
 			if (SpecifyConfiguration)
 				args.Add ($"/p:Configuration={projectConfiguration}");
+			if (Platform == TestPlatform.MacCatalyst)
+				args.Add ("/r");
 			args.Add (projectFile);
 			return args;
 		}

tests/xharness/Jenkins/TestTasks/MSBuildTask.cs

@@ -20,6 +20,8 @@ namespace Xharness.Jenkins.TestTasks {
 			get {
 				if (TestProject.IsDotNetProject) // 'dotnet build' will restore
 					return false;
+				if (TestProject.TestPlatform == TestPlatform.MacCatalyst)
+					return false; // we have to do 'msbuild /r'
 				return base.RestoreNugets;
 			}
 		}

tests/xharness/Jenkins/TestTasks/MacExecuteTask.cs

@@ -92,16 +92,26 @@ namespace Xharness.Jenkins.TestTasks {
 						var type = useXmlOutput ? LogType.XmlLog : LogType.NUnitResult;
 						xmlLog = Logs.Create ($"test-{Platform}-{Timestamp}.{extension}", type.ToString ());
 						arguments.Add ($"-transport:FILE");
+						proc.StartInfo.EnvironmentVariables ["NUNIT_TRANSPORT"] = "FILE";
 						arguments.Add ($"--logfile:{xmlLog.FullPath}");
+						proc.StartInfo.EnvironmentVariables ["NUNIT_LOG_FILE"] = xmlLog.FullPath;
 						if (useXmlOutput) {
 							arguments.Add ("--enablexml");
+							proc.StartInfo.EnvironmentVariables ["NUNIT_ENABLE_XML_OUTPUT"] = "true";
 							arguments.Add ("--xmlmode=wrapped");
+							proc.StartInfo.EnvironmentVariables ["NUNIT_ENABLE_XML_MODE"] = "wrapped";
 							arguments.Add ("--xmlversion=nunitv3");
+							proc.StartInfo.EnvironmentVariables ["NUNIT_XML_VERSION"] = "nunitv3";
 						}
+						arguments.Add ("--autostart");
+						proc.StartInfo.EnvironmentVariables ["NUNIT_AUTOSTART"] = "true";
+						arguments.Add ("--autoexit");
+						proc.StartInfo.EnvironmentVariables ["NUNIT_AUTOEXIT"] = "true";
 					}
 					if (!Harness.GetIncludeSystemPermissionTests (Platform, false))
 						proc.StartInfo.EnvironmentVariables ["DISABLE_SYSTEM_PERMISSION_TESTS"] = "1";
 					proc.StartInfo.EnvironmentVariables ["MONO_DEBUG"] = "no-gdb-backtrace";
+					proc.StartInfo.EnvironmentVariables.Remove ("DYLD_FALLBACK_LIBRARY_PATH"); // VSMac might set this, and the test may end up crashing
 					proc.StartInfo.Arguments = StringUtils.FormatArguments (arguments);
 					Jenkins.MainLog.WriteLine ("Executing {0} ({1})", TestName, Mode);
 					var log = Logs.Create ($"execute-{Platform}-{Timestamp}.txt", LogType.ExecutionLog.ToString ());

tests/xharness/Jenkins/TestTasks/MacTask.cs

@@ -20,6 +20,8 @@ namespace Xharness.Jenkins.TestTasks {
 					return "Mac Full";
 				case TestPlatform.Mac_System:
 					return "Mac System";
+				case TestPlatform.MacCatalyst:
+					return "Mac Catalyst";
 				default:
 					throw new NotImplementedException (Platform.ToString ());
 				}

tests/xharness/Targets/MacCatalystTarget.cs

@@ -0,0 +1,149 @@
+using System;
+using System.IO;
+using System.Xml;
+
+using Microsoft.DotNet.XHarness.iOS.Shared.Hardware;
+using Microsoft.DotNet.XHarness.iOS.Shared.Utilities;
+
+namespace Xharness.Targets {
+	public class MacCatalystTarget : UnifiedTarget {
+		public override string DotNetSdk => throw new NotImplementedException ();
+
+		public override string RuntimeIdentifier => throw new NotImplementedException ();
+
+		public override DevicePlatform ApplePlatform => throw new NotImplementedException ();
+
+		public override string PlatformString => "MacCatalyst";
+
+		public override string TargetFramework => "net6.0-maccatalyst";
+
+		public override string TargetFrameworkForNuGet => throw new NotImplementedException ();
+
+		public override string Suffix {
+			get {
+				return "-maccatalyst";
+			}
+		}
+
+		public override string ExtraLinkerDefsSuffix {
+			get {
+				return "-maccatalyst";
+			}
+		}
+
+		public override string ProjectFileSuffix {
+			get {
+				return Suffix;
+			}
+		}
+
+		protected override string TargetFrameworkIdentifier {
+			get {
+				return "Xamarin.MacCatalyst";
+			}
+		}
+
+		protected override string Imports {
+			get {
+				return IsFSharp ? "MacCatalyst\\Xamarin.MacCatalyst.FSharp.targets" : "MacCatalyst\\Xamarin.MacCatalyst.CSharp.targets";
+			}
+		}
+
+		protected override string BindingsImports {
+			get {
+				return IsFSharp ? "MacCatalyst\\Xamarin.MacCatalyst.ObjCBinding.FSharp.targets" : "MacCatalyst\\Xamarin.MacCatalyst.ObjCBinding.CSharp.targets";
+			}
+		}
+
+		public override string SimulatorArchitectures {
+			get {
+				return "x86_64";
+			}
+		}
+
+		public override string DeviceArchitectures {
+			get {
+				return "NotApplicable";
+			}
+		}
+
+		protected override string GetMinimumOSVersion (string templateMinimumOSVersion)
+		{
+			return "10.15";
+		}
+
+		protected override int [] UIDeviceFamily {
+			get {
+				return new int [] { 2 };
+			}
+		}
+
+		protected override string AdditionalDefines {
+			get {
+				return "XAMCORE_3_0;__MACCATALYST__";
+			}
+		}
+
+		public override string Platform {
+			get {
+				return "maccatalyst";
+			}
+		}
+
+		protected override bool SupportsBitcode {
+			get {
+				return false;
+			}
+		}
+
+		protected override void ProcessProject ()
+		{
+			base.ProcessProject ();
+
+			inputProject.ResolveAllPaths (TemplateProjectPath);
+
+			// I couldn't figure out why the NUnit package reference doesn't end up referencing nunit.framework.dll, so instead reference nunit.framework.dll directly
+			var itemGroup = inputProject.SelectNodes ("//*[local-name() = 'ItemGroup' and not(@Condition)]") [0];
+			var xmlElement = AddElement (inputProject, itemGroup, "PackageReference");
+			AddAttribute (inputProject, xmlElement, "Include", "NUnit");
+			AddAttribute (inputProject, xmlElement, "Version", "3.12.0");
+			AddAttribute (inputProject, xmlElement, "GeneratePathProperty", "true");
+
+			xmlElement = AddElement (inputProject, itemGroup, "Reference");
+			AddAttribute (inputProject, xmlElement, "Include", "$(PkgNUnit)\\lib\\netstandard2.0\\nunit.framework.dll");
+		}
+
+		static XmlElement AddElement (XmlDocument doc, XmlNode parent, string element)
+		{
+			var xmlElement = doc.CreateElement (element, doc.GetNamespace ());
+			parent.AppendChild (xmlElement);
+			return xmlElement;
+		}
+
+		static void AddAttribute (XmlDocument doc, XmlNode node, string attribute, string value)
+		{
+			var attrib = doc.CreateAttribute (attribute);
+			attrib.Value = value;
+			node.Attributes.Append (attrib);
+		}
+
+		protected override void UpdateInfoPList ()
+		{
+			var info_plist = new XmlDocument ();
+			var target_info_plist = Path.Combine (TargetDirectory, "Info" + Suffix + ".plist");
+			info_plist.LoadWithoutNetworkAccess (Path.Combine (TargetDirectory, OriginalInfoPListInclude));
+			BundleIdentifier = info_plist.GetCFBundleIdentifier ();
+
+			// Remove MinimumOSVersion
+			var node = info_plist.SelectSingleNode ("//dict/key[text()='MinimumOSVersion']");
+			node.ParentNode.RemoveChild (node.NextSibling);
+			node.ParentNode.RemoveChild (node);
+
+			// Add LSMinimumSystemVersion
+			info_plist.SetMinimummacOSVersion (GetMinimumOSVersion (null));
+
+			info_plist.SetUIDeviceFamily (UIDeviceFamily);
+			info_plist.Save (target_info_plist, Harness);
+		}
+	}
+}

tests/xharness/Targets/Target.cs

@@ -91,7 +91,7 @@ namespace Xharness.Targets
 			return rv.TrimStart ('-');
 		}
 
-		public string PlatformString {
+		public virtual string PlatformString {
 			get {
 				return ApplePlatform.AsString ();
 			}

tests/xharness/TestPlatform.cs

@@ -12,6 +12,7 @@
 		watchOS,
 		watchOS_32,
 		watchOS_64_32,
+		MacCatalyst,
 
 		Mac,
 		Mac_Modern,

tests/xharness/TestProjectExtensions.cs

@@ -44,6 +44,16 @@ namespace Xharness {
 			return clone;
 		}
 
+		public static TestProject AsMacCatalystProject (this TestProject self)
+		{
+			var clone = self.Clone ();
+			var suffix = string.Empty;
+			if (self.IsDotNetProject)
+				suffix = "-dotnet";
+			clone.Path = Path.Combine (Path.GetDirectoryName (self.Path), Target.ProjectsDir, "maccatalyst" + suffix, Path.GetFileNameWithoutExtension (self.Path) + "-maccatalyst" + Path.GetExtension (self.Path));
+			return clone;
+		}
+
 		// Get the referenced today extension project (if any)
 		public static TestProject GetTodayExtension (this TestProject self)
 		{

tests/xharness/iOSTestProject.cs

@@ -10,6 +10,7 @@ namespace Xharness {
 		public bool SkipwatchOSARM64_32Variation;
 		public bool SkipwatchOS32Variation;
 		public bool SkiptvOSVariation;
+		public bool SkipMacCatalystVariation;
 		public bool SkipTodayExtensionVariation;
 		public bool SkipDeviceVariations;
 		public bool BuildOnly;

tests/xharness/xharness.csproj

@@ -177,6 +177,7 @@
     <Compile Include="TestProjectExtensions.cs" />
     <Compile Include="TestTargetExtensions.cs" />
     <Compile Include="XmlDocumentExtensions.cs" />
+    <Compile Include="Targets\MacCatalystTarget.cs" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\..\tools\common\SdkVersions.cs">