[testing] Run tests on device (#17808)

* [tests] Run tests on real devices * [ios] Allow cake to run on device * [tests] Get android devices * fix xharness version * Clear api version * [tests] Find the correct android device to install * [tests] Correct variable name * [tests] Fix platform version * Fix variable * Fix merge
2023-11-28 13:21:29 +00:00 · 2023-11-28 13:21:29 +00:00 · 73d2142b14
--- a/eng/devices/android.cake
+++ b/eng/devices/android.cake
@ -6,13 +6,13 @@

 #tool nuget:?package=NUnit.ConsoleRunner&version=3.16.3

-const string defaultVersion = "30";
+const int defaultVersion = 30;
 const string dotnetVersion = "net8.0";

 // required
 FilePath PROJECT = Argument("project", EnvironmentVariable("ANDROID_TEST_PROJECT") ?? DEFAULT_PROJECT);
 string TEST_DEVICE = Argument("device", EnvironmentVariable("ANDROID_TEST_DEVICE") ?? $"android-emulator-64_{defaultVersion}");
-string DEVICE_NAME = Argument("skin", EnvironmentVariable("ANDROID_TEST_SKIN") ?? "Nexus 5X");
+string DEVICE_SKIN = Argument("skin", EnvironmentVariable("ANDROID_TEST_SKIN") ?? "Nexus 5X");

 // optional
 var USE_DOTNET = Argument("dotnet", true);
@ -28,13 +28,17 @@ var TEST_APP_INSTRUMENTATION = Argument("instrumentation", EnvironmentVariable("
 var TEST_RESULTS = Argument("results", EnvironmentVariable("ANDROID_TEST_RESULTS") ?? "");

 string TEST_WHERE = Argument("where", EnvironmentVariable("NUNIT_TEST_WHERE") ?? $"");
-var androidVersion = Argument("apiversion", EnvironmentVariable("ANDROID_PLATFORM_VERSION") ?? defaultVersion);
+
+string DEVICE_UDID = "";
+string DEVICE_VERSION = "";
+string DEVICE_NAME = "";
+string DEVICE_OS = "";

 // other
 string CONFIGURATION = Argument("configuration", "Debug");
 string TEST_FRAMEWORK = "net472";
 string ANDROID_AVD = "DEVICE_TESTS_EMULATOR";
-string DEVICE_ID = "";
+string ANDROID_AVD_IMAGE = "";
 string DEVICE_ARCH = "";
 bool DEVICE_BOOT = Argument("boot", true);
 bool DEVICE_BOOT_WAIT = Argument("wait", true);
@ -72,7 +76,7 @@ Setup(context =>
 	{
 		var working = TEST_DEVICE.Trim().ToLower();
 		var emulator = true;
-		var api = 30;
+		var api = defaultVersion;
 		// version
 		if (working.IndexOf("_") is int idx && idx > 0) {
 			api = int.Parse(working.Substring(idx + 1));
@ -107,27 +111,33 @@ Setup(context =>
 		var sdk = api >= 27 ? "google_apis_playstore" : "google_apis";
 		if (api == 27 && DEVICE_ARCH == "x86_64")
 			sdk = "default";
-		DEVICE_ID = $"system-images;android-{api};{sdk};{DEVICE_ARCH}";
+		ANDROID_AVD_IMAGE = $"system-images;android-{api};{sdk};{DEVICE_ARCH}";

-		Information("Going to run image: {0}", DEVICE_ID);
+		Information("Going to run image: {0}", ANDROID_AVD_IMAGE);
 		// we are not using a virtual device, so quit
 		if (!emulator)
+		{
+			Information("Not using a virtual device, skipping... and getting devices ");
+			
+			GetDevices(api.ToString());
+
 			return;
+		}
 	}

-	Information("Test Device ID: {0}", DEVICE_ID);
+	Information("Test Device ID: {0}", ANDROID_AVD_IMAGE);

 	if (DEVICE_BOOT) {
 		Information("Trying to boot the emulator...");

 		// delete the AVD first, if it exists
-		Information("Deleting AVD if exists: {0}...", ANDROID_AVD);
+		Information("Deleting AVD if exists: {0}...", ANDROID_AVD);	
 		try { AndroidAvdDelete(ANDROID_AVD, avdSettings); }
 		catch { }

 		// create the new AVD
 		Information("Creating AVD: {0}...", ANDROID_AVD);
-		AndroidAvdCreate(ANDROID_AVD, DEVICE_ID, DEVICE_NAME, force: true, settings: avdSettings);
+		AndroidAvdCreate(ANDROID_AVD, ANDROID_AVD_IMAGE, DEVICE_SKIN, force: true, settings: avdSettings);

 		// start the emulator
 		Information("Starting Emulator: {0}...", ANDROID_AVD);
@ -433,13 +443,18 @@ void SetupAppPackageNameAndResult()

 void InstallApk(string testApp, string testAppPackageName, string testResultsDirectory)
 {
+	var installadbSettings = new AdbToolSettings { SdkRoot = ANDROID_SDK_ROOT };
+	if(!string.IsNullOrEmpty(DEVICE_UDID))
+	{
+		installadbSettings.Serial = DEVICE_UDID;
+	}
 	if (DEVICE_BOOT_WAIT) {
 		Information("Waiting for the emulator to finish booting...");

 		// wait for it to finish booting (10 mins)
 		var waited = 0;
 		var total = 60 * 10;
-		while (AdbShell("getprop sys.boot_completed", adbSettings).FirstOrDefault() != "1") {
+		while (AdbShell("getprop sys.boot_completed", installadbSettings).FirstOrDefault() != "1") {
 			System.Threading.Thread.Sleep(1000);
 			Information("Wating {0}/{1} seconds for the emulator to boot up.", waited, total);
 			if (waited++ > total)
@ -449,20 +464,89 @@ void InstallApk(string testApp, string testAppPackageName, string testResultsDir
 	}

 	Information("Setting the ADB properties...");
-	var lines = AdbShell("setprop debug.mono.log default,mono_log_level=debug,mono_log_mask=all", adbSettings);
+	var lines = AdbShell("setprop debug.mono.log default,mono_log_level=debug,mono_log_mask=all", installadbSettings);
 	Information("{0}", string.Join("\n", lines));
-	lines = AdbShell("getprop debug.mono.log", adbSettings);
+	lines = AdbShell("getprop debug.mono.log", installadbSettings);
 	Information("{0}", string.Join("\n", lines));

-	//install apk on the emulator
+	//install apk on the emulator or device
 	Information("Install with xharness: {0}", testApp);
 	var settings = new DotNetToolSettings {
 		DiagnosticOutput = true,
-		ArgumentCustomization = args=>args.Append("run xharness android install " +
-			$"--app=\"{testApp}\" " +
-			$"--package-name=\"{testAppPackageName}\" " +
-			$"--output-directory=\"{testResultsDirectory}\" " +
-			$"--verbosity=\"Debug\" ")
+		ArgumentCustomization =  args =>
+						{
+							args.Append("run xharness android install " +
+										$"--app=\"{testApp}\" " +
+										$"--package-name=\"{testAppPackageName}\" " +
+										$"--output-directory=\"{testResultsDirectory}\" " +
+										$"--verbosity=\"Debug\" ");
+							
+							//if we specify a device we need to pass it to xharness
+							if(!string.IsNullOrEmpty(DEVICE_UDID))
+							{
+								args.Append($"--device-id=\"{DEVICE_UDID}\" ");
+							}
+
+							return args;
+						}
+	};
+	
+	if(!string.IsNullOrEmpty(DEVICE_UDID))
+	{
+		SetEnvironmentVariable("DEVICE_UDID", DEVICE_UDID);
+		//this needs to be translated to android 10/11 for appium
+		var realApi ="";
+		if(DEVICE_VERSION == "33")
+		{
+			realApi = "13";
+		}
+		if(DEVICE_VERSION == "32" || DEVICE_VERSION == "31")
+		{
+			realApi = "12";
+		}
+		else if(DEVICE_VERSION == "30")
+		{
+			realApi = "11";
+		}
+		SetEnvironmentVariable("PLATFORM_VERSION", realApi);
+	}
+
+	DotNetTool("tool", settings);
+}
+
+void GetDevices(string version)
+{
+	var deviceUdid = "";
+	var deviceName = "";
+	var deviceVersion = "";
+	var deviceOS = "";
+
+	var devices = AdbDevices(adbSettings);
+	foreach	(var device in devices)
+	{
+		deviceUdid = device.Serial;
+		deviceName = device.Model;
+		deviceOS = device.Product;
+
+		deviceVersion = AdbShell($"getprop ro.build.version.sdk ", new AdbToolSettings { SdkRoot = ANDROID_SDK_ROOT, Serial = deviceUdid }).FirstOrDefault();
+		Information("DeviceName:{0} udid:{1} version:{2} os:{3}", deviceName, deviceUdid, deviceVersion, deviceOS);
+
+		if(version.Contains(deviceVersion.Split(".")[0]))
+		{
+			Information("We want this device: {0} {1} because it matches {2}", deviceName, deviceVersion, version);
+			DEVICE_UDID = deviceUdid;
+			DEVICE_VERSION = deviceVersion;
+			DEVICE_NAME = deviceName;
+			DEVICE_OS = deviceOS;
+			break;
+		}
+	}
+
+	//this will fail if there are no devices with this api attached
+	var settings = new DotNetToolSettings {
+			DiagnosticOutput = true,
+			ArgumentCustomization = args=>args.Append("run xharness android device " +
+			$"--api-version=\"{version}\" " )
 	};
 	DotNetTool("tool", settings);
 }
--- a/eng/devices/devices-shared.cake
+++ b/eng/devices/devices-shared.cake
@ -9,6 +9,12 @@ if (string.Equals(TARGET, "uitest", StringComparison.OrdinalIgnoreCase))
    DEFAULT_APP_PROJECT = "../../src/Controls/samples/Controls.Sample.UITests/Controls.Sample.UITests.csproj";
 }

+if (string.Equals(TARGET, "uitest-build", StringComparison.OrdinalIgnoreCase))
+{
+    DEFAULT_PROJECT = "../../src/Controls/tests/UITests/Controls.AppiumTests.csproj";
+    DEFAULT_APP_PROJECT = "../../src/Controls/samples/Controls.Sample.UITests/Controls.Sample.UITests.csproj";
+}
+
 if (string.Equals(TARGET, "cg-uitest", StringComparison.OrdinalIgnoreCase))
 {
    DEFAULT_PROJECT = "../../src/Compatibility/ControlGallery/test/iOS.UITests/Compatibility.ControlGallery.iOS.UITests.csproj";
--- a/eng/devices/ios.cake
+++ b/eng/devices/ios.cake
@ -143,6 +143,35 @@ Task("Build")
 		});
 });

+Task("uitest-build")
+	.Does(() =>
+{
+	var name = System.IO.Path.GetFileNameWithoutExtension(DEFAULT_APP_PROJECT);
+	var binlog = $"{BINLOG_DIR}/{name}-{CONFIGURATION}-ios.binlog";
+
+	Information("app" +DEFAULT_APP_PROJECT);
+	DotNetBuild(DEFAULT_APP_PROJECT, new DotNetBuildSettings {
+		Configuration = CONFIGURATION,
+		Framework = TARGET_FRAMEWORK,
+		ToolPath = DOTNET_PATH,
+		ArgumentCustomization = args =>
+		{ 	
+			args
+			.Append("/p:BuildIpa=true")
+			.Append("/p:RuntimeIdentifier=ios-arm64")
+			.Append("/bl:" + binlog);
+			
+			// if we building for a device
+			if(TEST_DEVICE.ToLower().Contains("device"))
+			{
+				args.Append("/p:RuntimeIdentifier=ios-arm64");
+			}
+
+			return args;
+		}
+	});
+});
+
 Task("Test")
 	.IsDependentOn("Build")
 	.Does(() =>
@ -230,6 +259,7 @@ Task("Test")
 });

 Task("uitest")
+	.IsDependentOn("uitest-build")
 	.Does(() =>
 {
 	SetupAppPackageNameAndResult();
@ -353,10 +383,9 @@ void InstallIpa(string testApp, string testAppPackageName, string testDevice, st
 	try {
 		DotNetTool("tool", settings);
 	} finally {
-
 		string iosVersionToRun = version;
 		string deviceToRun = "";	
-
+		
 		if (testDevice.Contains("device"))
 		{	
 			if(!string.IsNullOrEmpty(DEVICE_UDID))
@ -367,9 +396,9 @@ void InstallIpa(string testApp, string testAppPackageName, string testDevice, st
 			{
 				throw new Exception("No device was found to run tests on.");
 			}
-
+			
 			iosVersionToRun = DEVICE_VERSION;
-
+			
 			Information("The device to run tests: {0} {1}", DEVICE_NAME, iosVersionToRun);
 		}
 		else
@ -382,13 +411,14 @@ void InstallIpa(string testApp, string testAppPackageName, string testDevice, st
 			var simXH = sims.Where(s => s.Name.Contains(simulatorName)).FirstOrDefault();
 			if(simXH == null)
 				throw new Exception("No simulator was found to run tests on.");
+
 			deviceToRun = simXH.UDID;
 			DEVICE_NAME = simXH.Name;
 			Information("The emulator to run tests: {0} {1}", simXH.Name, simXH.UDID);
 		}
+
 		Information("The platform version to run tests: {0}", iosVersionToRun);
 		SetEnvironmentVariable("DEVICE_UDID", deviceToRun);
-		SetEnvironmentVariable("DEVICE_NAME", DEVICE_NAME);
 		SetEnvironmentVariable("PLATFORM_VERSION", iosVersionToRun);
 	}
 }
@ -397,7 +427,7 @@ void GetSimulators(string version)
 {
 	DotNetTool("tool", new DotNetToolSettings {
 			ToolPath = DOTNET_TOOL_PATH,
-			DiagnosticOutput = true,	
+			DiagnosticOutput = true,
 			ArgumentCustomization = args => args.Append("run xharness apple simulators install " +
 				$"\"{version}\" " +
 				$"--verbosity=\"Debug\" ")
@ -410,7 +440,7 @@ void GetDevices(string version)
 	var deviceName = "";
 	var deviceVersion = "";
 	var deviceOS = "";
-
+	
 	var list = new List<string>();
 	bool isDevice = false;
 	// print the apple state of the machine
@ -468,4 +498,4 @@ void GetDevices(string version)
 			break;
 		}
 	}
-}
+}
--- a/eng/scripts/appium-install.ps1
+++ b/eng/scripts/appium-install.ps1
@ -41,7 +41,7 @@ param
    [string] $appiumVersion = '2.1.1',
    [string] $windowsDriverVersion = '2.10.1',
    [string] $androidDriverVersion = '2.29.4',
-    [string] $iOSDriverVersion = '4.34.0',
+    [string] $iOSDriverVersion = '5.2.0',
    [string] $macDriverVersion = '1.7.2'
 )

--- a/src/Controls/samples/Controls.Sample.UITests/Controls.Sample.UITests.csproj
+++ b/src/Controls/samples/Controls.Sample.UITests/Controls.Sample.UITests.csproj
@ -46,18 +46,6 @@
    <MauiSplashScreen Include="Resources\Splash\splash.svg" Color="#FFFFFF" BaseSize="168,208" />
  </ItemGroup>

-  <ItemGroup>
-    <Compile Update="Elements\BordersWithVariousShapes.xaml.cs">
-      <DependentUpon>BordersWithVariousShapes.xaml</DependentUpon>
-    </Compile>
-  </ItemGroup>
-
  <Import Project="$(MauiSrcDirectory)Maui.InTree.props" Condition=" '$(UseMaui)' != 'true' " />

-  <ItemGroup>
-    <MauiXaml Update="Elements\BordersWithVariousShapes.xaml">
-      <Generator>MSBuild:Compile</Generator>
-    </MauiXaml>
-  </ItemGroup>
-
 </Project>
--- a/src/Controls/tests/UITests/UITest.cs
+++ b/src/Controls/tests/UITests/UITest.cs
@ -47,7 +47,7 @@ namespace Microsoft.Maui.AppiumTests

 		public override IConfig GetTestConfig()
 		{
-			var frameworkVersion = "net7.0";
+			var frameworkVersion = "net8.0";
 #if DEBUG
 			var configuration = "Debug";
 #else
@ -59,6 +59,10 @@ namespace Microsoft.Maui.AppiumTests

 			switch (_testDevice)
 			{
+				case TestDevice.Android:
+					config.SetProperty("PlatformVersion", Environment.GetEnvironmentVariable("PLATFORM_VERSION") ?? "");
+					config.SetProperty("Udid", Environment.GetEnvironmentVariable("DEVICE_UDID") ?? "");
+					break;
 				case TestDevice.iOS:
 					config.SetProperty("DeviceName", Environment.GetEnvironmentVariable("DEVICE_NAME") ?? "iPhone X");
 					config.SetProperty("PlatformVersion", Environment.GetEnvironmentVariable("PLATFORM_VERSION") ?? "17.0");