[xharness] Generate a system variant of dont link, and run it using the oldest mono version we support in Jenkins. Fixes #4121. (#4968)

This is an addition to the tests we already run on older macOS bots (and as such will not execute on our PR bots).

Fixes https://github.com/xamarin/xamarin-macios/issues/4121.

jenkins/Jenkinsfile

@@ -309,6 +309,24 @@ def runXamarinMacTests (url, macOS, maccore_hash, xamarin_macios_hash)
                     failedTests.add (t)
                 }
             }
+            // Run dontlink using the oldest system mono we support
+            def t = "dontlink (system)"
+            try {
+                // install oldest supported mono
+                sh ('''
+                    cd mac-test-package
+                    URL=`grep "^MIN_XM_MONO_URL=" Make.config | sed 's/.*=//'`
+                    curl -L "$URL" --output old-mono.pkg
+                    sudo installer -pkg old-mono.pkg -target /
+                    mono --version
+                    ''')
+                // run dontlink tests using the system mono
+                sh ("make -C mac-test-package/tests exec-mac-system-dontlink")
+            } catch (error) {
+                echoError ("${t} failed with error: ${error}")
+                failed = true
+                failedTests.add (t)
+            }
         }
     } finally {
         sh ("rm -rf ${workspace}/mac-test-package ${workspace}/*.zip")

tests/.gitignore

@@ -21,6 +21,7 @@ build
 *-today.?sproj
 *-today-extension.?sproj
 *-today.sln
+*-system.csproj
 Makefile-mac.inc
 .stamp*
 Info-*.plist

tests/package-mac-tests.sh

@@ -8,6 +8,7 @@ rm -Rf $DIR
 mkdir -p $DIR
 
 make build-mac
+make build-mac-system-dontlink
 
 for app in */bin/x86/*/*.app linker/mac/*/bin/x86/*/*.app introspection/Mac/bin/x86/*/*.app; do
 	mkdir -p "$DIR/tests/$app"

tests/xharness/BCLTestInfo.cs

@@ -174,7 +174,7 @@ namespace xharness
 
 		public MacBCLTestInfo (Harness harness, string testName, MacFlavors flavor) : base (harness, testName)
 		{
-			if (flavor == MacFlavors.All)
+			if (flavor == MacFlavors.All || flavor == MacFlavors.NonSystem)
 				throw new ArgumentException ("Each target must be a specific flavor");
 
 			Flavor = flavor;
@@ -204,6 +204,8 @@ namespace xharness
 			case MacFlavors.Full:
 				inputProject.AddAdditionalDefines ("XAMMAC_4_5");
 				break;
+			default:
+				throw new NotImplementedException (Flavor.ToString ());
 			}
 			inputProject.SetOutputPath ("bin\\$(Platform)\\$(Configuration)" + FlavorSuffix);
 			inputProject.SetIntermediateOutputPath ("obj\\$(Platform)\\$(Configuration)" + FlavorSuffix);

tests/xharness/Harness.cs

@@ -227,11 +227,15 @@ namespace xharness
 		void AutoConfigureMac ()
 		{
 			var test_suites = new [] {
-				new { Directory = "apitest", ProjectFile = "apitest", Name = "apitest" },
+				new { Directory = "apitest", ProjectFile = "apitest", Name = "apitest", GenerateSystem = false },
-				new { Directory = "linker/mac/dont link", ProjectFile = "dont link-mac", Name = "dont link" },
+				new { Directory = "linker/mac/dont link", ProjectFile = "dont link-mac", Name = "dont link", GenerateSystem = true },
 			};
-			foreach (var p in test_suites)
+			foreach (var p in test_suites) {
-				MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p.Directory + "/" + p.ProjectFile + ".sln"))) { Name = p.Name });
+				MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (RootDirectory, p.Directory + "/" + p.ProjectFile + ".sln"))) {
+					Name = p.Name,
+					TargetFrameworkFlavor = p.GenerateSystem ? MacFlavors.All : MacFlavors.NonSystem,
+				});
+			}
 			
 			MacTestProjects.Add (new MacTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "introspection", "Mac", "introspection-mac.csproj")), targetFrameworkFlavor: MacFlavors.Modern) { Name = "introspection" });
 
@@ -449,6 +453,13 @@ namespace xharness
 					}
 				}
 
+				if (proj.GenerateSystem) {
+					var system = new MacUnifiedTarget (false, false);
+					system.System = true;
+					configureTarget (system, file, proj.IsNUnitProject);
+					unified_targets.Add (system);
+				}
+
 				var classic = new MacClassicTarget ();
 				configureTarget (classic, file, false);
 				classic_targets.Add (classic);

tests/xharness/Jenkins.cs

@@ -260,6 +260,7 @@ namespace xharness
 						case TestPlatform.Mac_Unified32:
 						case TestPlatform.Mac_UnifiedXM45:
 						case TestPlatform.Mac_UnifiedXM45_32:
+						case TestPlatform.Mac_UnifiedSystem:
 							isMac = true;
 							break;
 						}
@@ -750,6 +751,8 @@ namespace xharness
 								Tasks.Add (CloneExecuteTask (e, project, TestPlatform.Mac_UnifiedXM45, "-unifiedXM45", ignored));
 								Tasks.Add (CloneExecuteTask (e, project, TestPlatform.Mac_UnifiedXM45_32, "-unifiedXM45-32", ignored32, true));
 							}
+							if (project.GenerateSystem)
+								Tasks.Add (CloneExecuteTask (e, project, TestPlatform.Mac_UnifiedSystem, "-system", ignored));
 						}
 					}
 				}
@@ -1077,6 +1080,7 @@ namespace xharness
 										case TestPlatform.Mac_Unified32:
 										case TestPlatform.Mac_UnifiedXM45:
 										case TestPlatform.Mac_UnifiedXM45_32:
+										case TestPlatform.Mac_UnifiedSystem:
 											is_match = true;
 											break;
 										default:
@@ -2332,6 +2336,8 @@ function toggleAll (show)
 					return rv.Substring (0, rv.Length - "-unifiedXM45".Length);
 				case TestPlatform.Mac_UnifiedXM45_32:
 					return rv.Substring (0, rv.Length - "-unifiedXM45-32".Length);
+				case TestPlatform.Mac_UnifiedSystem:
+					return rv.Substring (0, rv.Length - "-unifiedSystem".Length);
 				default:
 					if (rv.EndsWith ("-watchos", StringComparison.Ordinal)) {
 						return rv.Substring (0, rv.Length - 8);
@@ -2481,6 +2487,7 @@ function toggleAll (show)
 			case TestPlatform.Mac_Unified32:
 			case TestPlatform.Mac_UnifiedXM45:
 			case TestPlatform.Mac_UnifiedXM45_32:
+			case TestPlatform.Mac_UnifiedSystem:
 				process.StartInfo.EnvironmentVariables ["MD_APPLE_SDK_ROOT"] = xcodeRoot;
 				process.StartInfo.EnvironmentVariables ["XBUILD_FRAMEWORK_FOLDERS_PATH"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild-frameworks");
 				process.StartInfo.EnvironmentVariables ["MSBuildExtensionsPath"] = Path.Combine (Harness.MAC_DESTDIR, "Library", "Frameworks", "Mono.framework", "External", "xbuild");
@@ -2947,8 +2954,10 @@ function toggleAll (show)
 					return "Mac Unified XM45";
 				case TestPlatform.Mac_UnifiedXM45_32:
 					return "Mac Unified XM45 32-bit";
+				case TestPlatform.Mac_UnifiedSystem:
+					return "Mac Unified System";
 				default:
-					throw new NotImplementedException ();
+					throw new NotImplementedException (Platform.ToString ());
 				}
 			}
 			set {
@@ -3014,6 +3023,9 @@ function toggleAll (show)
 			case TestPlatform.Mac_UnifiedXM45_32:
 				suffix = "-unifiedXM45-32";
 				break;
+			case TestPlatform.Mac_UnifiedSystem:
+				suffix = "-unifiedSystem";
+				break;
 			}
 			if (ProjectFile.EndsWith (".sln", StringComparison.Ordinal)) {
 				Path = System.IO.Path.Combine (System.IO.Path.GetDirectoryName (ProjectFile), "bin", BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration + suffix, name + ".app", "Contents", "MacOS", name);
@@ -3831,6 +3843,7 @@ function toggleAll (show)
 		Mac_UnifiedXM45,
 		Mac_Unified32,
 		Mac_UnifiedXM45_32,
+		Mac_UnifiedSystem,
 	}
 
 	[Flags]

tests/xharness/MacUnifiedTarget.cs

@@ -6,6 +6,7 @@ namespace xharness
 	public class MacUnifiedTarget : MacTarget
 	{
 		public bool Mobile { get; private set; }
+		public bool System { get; set; }
 
 		// Optional
 		public MacBCLTestInfo BCLInfo { get; set; }
@@ -42,13 +43,19 @@ namespace xharness
 			get {
 				if (SkipProjectGeneration)
 					return "";
-				string suffix = (Mobile ? "" : "XM45") + (ThirtyTwoBit ? "-32" : "");
+
+				if (System)
+					return "-system";
+
+				var suffix = (Mobile ? "" : "XM45") + (ThirtyTwoBit ? "-32" : "");
 				return "-unified" + (IsBCL ? "" : suffix);
 			}
 		}
 
 		public override string MakefileWhereSuffix {
 			get {
+				if (System)
+					return "system";
 				string suffix = (Mobile ? "" : "XM45") + (ThirtyTwoBit ? "32" : "");
 				return "unified" + (IsBCL ? "" : suffix);
 			}
@@ -115,7 +122,10 @@ namespace xharness
 				var props = new Dictionary<string, string> ();
 
 
-				if (Mobile)
+				if (System) {
+					props.Add ("TargetFrameworkVersion", "v4.7.1");
+					props.Add ("MonoBundlingExtraArgs", "--embed-mono=no");
+				} else if (Mobile)
 				{
 					props.Add ("TargetFrameworkVersion", "v2.0");
 				}

tests/xharness/TestProject.cs

@@ -160,7 +160,7 @@ namespace xharness
 		}
 	}
 
-	public enum MacFlavors { All, Modern, Full }
+	public enum MacFlavors { All, Modern, Full, System, NonSystem }
 
 	public class MacTestProject : TestProject
 	{
@@ -169,8 +169,9 @@ namespace xharness
 		// Optional
 		public MacBCLTestInfo BCLInfo { get; set; }
 
-		public bool GenerateModern => TargetFrameworkFlavor == MacFlavors.All || TargetFrameworkFlavor == MacFlavors.Modern;
+		public bool GenerateModern => TargetFrameworkFlavor == MacFlavors.All || TargetFrameworkFlavor == MacFlavors.NonSystem || TargetFrameworkFlavor == MacFlavors.Modern;
-		public bool GenerateFull => TargetFrameworkFlavor == MacFlavors.All || TargetFrameworkFlavor == MacFlavors.Full;
+		public bool GenerateFull => TargetFrameworkFlavor == MacFlavors.All || TargetFrameworkFlavor == MacFlavors.NonSystem || TargetFrameworkFlavor == MacFlavors.Full;
+		public bool GenerateSystem => TargetFrameworkFlavor == MacFlavors.All || TargetFrameworkFlavor == MacFlavors.System;
 
 		public string Platform = "x86";
 
@@ -178,7 +179,7 @@ namespace xharness
 		{
 		}
 
-		public MacTestProject (string path, bool isExecutableProject = true, bool generateVariations = true, MacFlavors targetFrameworkFlavor = MacFlavors.All) : base (path, isExecutableProject, generateVariations)
+		public MacTestProject (string path, bool isExecutableProject = true, bool generateVariations = true, MacFlavors targetFrameworkFlavor = MacFlavors.NonSystem) : base (path, isExecutableProject, generateVariations)
 		{
 			TargetFrameworkFlavor = targetFrameworkFlavor;
 		}