[tests] Add support to xharness for excluding tests that require permission dialogs, and update monotouch-test accordingly. (#2418)

* [monotouch-test] Optionally ignore tests that show permission dialogs.

* [xharness] Add support for excluding tests that require permission dialogs.

* [xharness] Ignore introspection tests if we don't want permission dialogs.

Ignore introspection tests for now if we don't want permission dialogs, since
figuring out which API requires permissions is a tedious process (which we'll
eventually have to do though).

tests/common/TestRuntime.cs

@@ -4,10 +4,20 @@ using System.Linq;
 using System.Runtime.InteropServices;
 
 #if XAMCORE_2_0
+using AVFoundation;
 using Foundation;
+#if !__TVOS__
+using Contacts;
+#endif
 #if MONOMAC
 using AppKit;
 #else
+#if !__TVOS__ && !__WATCHOS__
+using AddressBook;
+#endif
+#if !__WATCHOS__
+using MediaPlayer;
+#endif
 using UIKit;
 #endif
 using ObjCRuntime;
@@ -16,6 +26,7 @@ using ObjCRuntime;
 using MonoMac.ObjCRuntime;
 using MonoMac.Foundation;
 using MonoMac.AppKit;
+using MonoMac.AVFoundation;
 #else
 using MonoTouch.ObjCRuntime;
 using MonoTouch.Foundation;
@@ -393,4 +404,126 @@ partial class TestRuntime
 #endif
 		}
 	}
+
+	public static bool IgnoreTestThatRequiresSystemPermissions ()
+	{
+		return !string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("DISABLE_SYSTEM_PERMISSION_TESTS"));
+	}
+
+#if !MONOMAC && !__TVOS__ && !__WATCHOS__
+	public static void RequestCameraPermission (NSString mediaTypeToken, bool assert_granted = false)
+	{
+		if (AVCaptureDevice.GetAuthorizationStatus (mediaTypeToken) == AVAuthorizationStatus.NotDetermined) {
+			if (IgnoreTestThatRequiresSystemPermissions ())
+				NUnit.Framework.Assert.Ignore ("This test would show a dialog to ask for permission to access the camera.");
+
+			AVCaptureDevice.RequestAccessForMediaType (mediaTypeToken, (accessGranted) =>
+			{
+				Console.WriteLine ("Camera permission {0}", accessGranted ? "granted" : "denied");
+			});
+		}
+
+		switch (AVCaptureDevice.GetAuthorizationStatus (AVMediaType.Video)) {
+		case AVAuthorizationStatus.Restricted:
+		case AVAuthorizationStatus.Denied:
+			if (assert_granted)
+				NUnit.Framework.Assert.Fail ("This test requires permission to access the camera.");
+			break;
+		}
+	}
+#endif // !!MONOMAC && !__TVOS__ && !__WATCHOS__
+
+#if XAMCORE_2_0 && !__TVOS__
+	public static void CheckContactsPermission (bool assert_granted = false)
+	{
+		switch (CNContactStore.GetAuthorizationStatus (CNEntityType.Contacts)) {
+		case CNAuthorizationStatus.NotDetermined:
+			if (IgnoreTestThatRequiresSystemPermissions ())
+				NUnit.Framework.Assert.Ignore ("This test would show a dialog to ask for permission to access the contacts.");
+			// We don't request access here, because there's no global method to request access (an contact store instance is required).
+			// Interestingly there is a global method to determine if access has been granted...
+			break;
+		case CNAuthorizationStatus.Restricted:
+		case CNAuthorizationStatus.Denied:
+			if (assert_granted)
+				NUnit.Framework.Assert.Fail ("This test requires permission to access the contacts.");
+			break;
+		}
+	}
+#endif // XAMCORE_2_0
+
+#if !MONOMAC && !__TVOS__ && !__WATCHOS__
+	public static void CheckAddressBookPermission (bool assert_granted = false)
+	{
+		switch (ABAddressBook.GetAuthorizationStatus ()) {
+		case ABAuthorizationStatus.NotDetermined:
+			if (IgnoreTestThatRequiresSystemPermissions ())
+				NUnit.Framework.Assert.Ignore ("This test would show a dialog to ask for permission to access the address book.");
+			// We don't request access here, because there's no global method to request access (an addressbook instance is required).
+			// Interestingly there is a global method to determine if access has been granted...
+			break;
+		case ABAuthorizationStatus.Restricted:
+		case ABAuthorizationStatus.Denied:
+			if (assert_granted)
+				NUnit.Framework.Assert.Fail ("This test requires permission to access the address book.");
+			break;
+		}
+	}
+#endif // !MONOMAC && !__TVOS__ && !__WATCHOS__
+
+#if !__WATCHOS__
+	public static void RequestMicrophonePermission (bool assert_granted = false)
+	{
+#if MONOMAC
+		// It looks like macOS does not restrict access to the microphone.
+#elif __TVOS__
+		// tvOS doesn't have a (developer-accessible) microphone, but it seems to have API that requires developers 
+		// to request microphone access on other platforms (which means that it makes sense to both run those tests
+		// on tvOS (because the API's there) and to request microphone access (because that's required on other platforms).
+#else
+		if (!CheckXcodeVersion (6, 0))
+			return; // The API to check/request permission isn't available in earlier versions, the dialog will just pop up.
+
+		if (AVAudioSession.SharedInstance ().RecordPermission == AVAudioSessionRecordPermission.Undetermined) {
+			if (IgnoreTestThatRequiresSystemPermissions ())
+				NUnit.Framework.Assert.Ignore ("This test would show a dialog to ask for permission to access the microphone.");
+
+			AVAudioSession.SharedInstance ().RequestRecordPermission ((bool granted) =>
+			{
+				Console.WriteLine ("Microphone permission {0}", granted ? "granted" : "denied");
+			});
+		}
+
+		switch (AVAudioSession.SharedInstance ().RecordPermission) { // iOS 8+
+		case AVAudioSessionRecordPermission.Denied:
+			if (assert_granted)
+				NUnit.Framework.Assert.Fail ("This test requires permission to access the microphone.");
+			break;
+		}
+#endif // !MONOMAC && !__TVOS__
+	}
+#endif // !__WATCHOS__
+
+#if !MONOMAC && !__TVOS__ && !__WATCHOS__
+	public static void RequestMediaLibraryPermission (bool assert_granted = false)
+	{
+		if (MPMediaLibrary.AuthorizationStatus == MPMediaLibraryAuthorizationStatus.NotDetermined) {
+			if (IgnoreTestThatRequiresSystemPermissions ())
+				NUnit.Framework.Assert.Ignore ("This test would show a dialog to ask for permission to access the media library.");
+
+			MPMediaLibrary.RequestAuthorization ((access) =>
+			{
+				Console.WriteLine ("Media library permission: {0}", access);
+			});
+		}
+
+		switch (MPMediaLibrary.AuthorizationStatus) {
+		case MPMediaLibraryAuthorizationStatus.Denied:
+		case MPMediaLibraryAuthorizationStatus.Restricted:
+			if (assert_granted)
+				NUnit.Framework.Assert.Fail ("This test requires permission to access the media library.");
+			break;
+		}
+	}
+#endif // !MONOMAC && !__TVOS__
 }

tests/monotouch-test/AVFoundation/AudioRecorderTest.cs

@@ -39,6 +39,8 @@ namespace MonoTouchFixtures.AVFoundation {
 		[Test]
 		public void Create ()
 		{
+			TestRuntime.RequestMicrophonePermission ();
+
 			var url =  NSUrl.FromFilename ("/dev/null");
 			NSError error;
 			var audioSettings = new AudioSettings (NSDictionary.FromObjectsAndKeys (Values, Keys));
@@ -51,6 +53,8 @@ namespace MonoTouchFixtures.AVFoundation {
 		[Test]
 		public void CreateWithError ()
 		{
+			TestRuntime.RequestMicrophonePermission ();
+
 			var url =  NSUrl.FromFilename ("/dev/fake.wav");
 			NSError error;
 			var audioSettings = new AudioSettings (NSDictionary.FromObjectsAndKeys (Values, Keys));

tests/monotouch-test/AVFoundation/CaptureMetadataOutputTest.cs

@@ -90,13 +90,7 @@ namespace MonoTouchFixtures.AVFoundation {
 			if (Runtime.Arch != Arch.DEVICE)
 				Assert.Ignore ("This test only runs on device (requires camera access)");
 
-			var auth = AVCaptureDevice.GetAuthorizationStatus (AVMediaType.Video);
+			TestRuntime.RequestCameraPermission (AVMediaType.Video, true);
-			switch (auth) {
-			case AVAuthorizationStatus.Restricted:
-			case AVAuthorizationStatus.Denied:
-				Assert.Fail ("This test requires access to the camera, but the app has been denied access.");
-				break;
-			}
 
 			using (var captureSession = new AVCaptureSession ()) {
 				using (var videoDevice = AVCaptureDevice.DefaultDeviceWithMediaType (AVMediaType.Video)) {

tests/monotouch-test/AudioToolbox/AudioQueueTest.cs

@@ -31,6 +31,8 @@ namespace MonoTouchFixtures.AudioToolbox {
 		[Test]
 		public void Properties ()
 		{
+			TestRuntime.RequestMicrophonePermission ();
+
 			var b = new InputAudioQueue (AudioStreamBasicDescription.CreateLinearPCM ());
 			b.HardwareCodecPolicy = AudioQueueHardwareCodecPolicy.PreferHardware;
 

tests/monotouch-test/Contacts/ContactStoreTest.cs

@@ -41,6 +41,8 @@ namespace MonoTouchFixtures.Contacts {
 		{
 			string identifier = null;
 
+			TestRuntime.CheckContactsPermission ();
+
 			var fetchKeys = new [] { CNContactKey.Identifier, CNContactKey.GivenName, CNContactKey.FamilyName };
 			NSError error;
 			using (var predicate = CNContact.GetPredicateForContacts ("Appleseed"))

tests/monotouch-test/MediaPlayer/MediaItemTest.cs

@@ -27,6 +27,8 @@ namespace MonoTouchFixtures.MediaPlayer {
 			if (Runtime.Arch != Arch.DEVICE)
 				Assert.Inconclusive ("This test only works on device (the simulator does not have an iPod Music library).");
 
+			TestRuntime.RequestMediaLibraryPermission (true);
+
 			using (var q = new MPMediaQuery ()) {
 				var items = q.Items;
 				if (items == null)

tests/xharness/AppRunner.cs

@@ -75,6 +75,8 @@ namespace xharness
 			}
 		}
 
+		public bool IncludeSystemPermissionTests { get; set; } = true;
+
 		// For watch apps we end up with 2 simulators, the watch simulator (the main one), and the iphone simulator (the companion one).
 		SimDevice[] simulators;
 		SimDevice simulator { get { return simulators [0]; } }
@@ -477,6 +479,9 @@ namespace xharness
 			if (Harness.InJenkins) 
 				args.Append (" -setenv=NUNIT_ENABLE_XML_OUTPUT=true");
 
+			if (!IncludeSystemPermissionTests)
+				args.Append (" -setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1");
+
 			if (isSimulator) {
 				args.Append (" -argument=-app-arg:-hostname:127.0.0.1");
 				args.Append (" -setenv=NUNIT_HOSTNAME=127.0.0.1");

tests/xharness/Jenkins.cs

@@ -30,6 +30,7 @@ namespace xharness
 		public bool IncludeMacBindingProject;
 		public bool IncludeSimulator = true;
 		public bool IncludeDevice;
+		public bool IncludeSystemPermissionTests = true; // tests that require access to system resources (system contacts, photo library, etc) in order to work
 
 		public Logs Logs = new Logs ();
 		public Log MainLog;
@@ -108,6 +109,20 @@ namespace xharness
 			return runtasks;
 		}
 
+		bool IsIncluded (TestProject project)
+		{
+			if (!project.IsExecutableProject)
+				return false;
+
+			if (!IncludeBcl && project.IsBclTest)
+				return false;
+
+			if (!IncludeSystemPermissionTests && project.Name == "introspection")
+				return false;
+
+			return true;
+		}
+
 		async Task<IEnumerable<TestTask>> CreateRunDeviceTasks ()
 		{
 			var rv = new List<RunDeviceTask> ();
@@ -123,11 +138,8 @@ namespace xharness
 			}
 
 			foreach (var project in Harness.IOSTestProjects) {
-				if (!project.IsExecutableProject)
-					continue;
-
 				bool ignored = !IncludeDevice;
-				if (!IncludeBcl && project.IsBclTest)
+				if (!IsIncluded (project))
 					ignored = true;
 				
 				var build64 = new XBuildTask
@@ -389,6 +401,7 @@ namespace xharness
 			SetEnabled (labels, "mac-classic", ref IncludeClassicMac);
 			SetEnabled (labels, "ios-msbuild", ref IncludeiOSMSBuild);
 			SetEnabled (labels, "ios-simulator", ref IncludeSimulator);
+			SetEnabled (labels, "system-permission", ref IncludeSystemPermissionTests);
 		}
 
 		void SetEnabled (HashSet<string> labels, string testname, ref bool value)
@@ -419,14 +432,11 @@ namespace xharness
 			var runSimulatorTasks = new List<RunSimulatorTask> ();
 
 			foreach (var project in Harness.IOSTestProjects) {
-				if (!project.IsExecutableProject)
-					continue;
-
 				bool ignored = false;
 				if (!IncludeSimulator)
 					ignored = true;
 
-				if (!IncludeBcl && project.IsBclTest)
+				if (!IsIncluded (project))
 					ignored = true;
 
 				var ps = new List<Tuple<TestProject, TestPlatform, bool>> ();
@@ -475,14 +485,11 @@ namespace xharness
 			Tasks.Add (nunitExecutioniOSMSBuild);
 			
 			foreach (var project in Harness.MacTestProjects) {
-				if (!project.IsExecutableProject)
-					continue;
-
 				bool ignored = !IncludeMac;
 				if (!IncludeMmpTest && project.Path.Contains ("mmptest"))
 					ignored = true;
 
-				if (!IncludeBcl && project.IsBclTest)
+				if (!IsIncluded (project))
 					ignored = true;
 
 				var configurations = project.Configurations;
@@ -2721,6 +2728,7 @@ function oninitialload ()
 						DeviceName = Device.Name,
 						CompanionDeviceName = CompanionDevice?.Name,
 						Configuration = ProjectConfiguration,
+						IncludeSystemPermissionTests = Jenkins.IncludeSystemPermissionTests,
 					};
 
 					// Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first.
@@ -2770,6 +2778,7 @@ function oninitialload ()
 								DeviceName = Device.Name,
 								CompanionDeviceName = CompanionDevice?.Name,
 								Configuration = ProjectConfiguration,
+								IncludeSystemPermissionTests = Jenkins.IncludeSystemPermissionTests,
 							};
 							additional_runner = todayRunner;
 							await todayRunner.RunAsync ();
@@ -2856,6 +2865,7 @@ function oninitialload ()
 				Target = AppRunnerTarget,
 				LogDirectory = LogDirectory,
 				MainLog = Logs.CreateStream (LogDirectory, $"run-{Device.UDID}-{Timestamp}.log", "Run log"),
+				IncludeSystemPermissionTests = Jenkins.IncludeSystemPermissionTests,
 			};
 			runner.Simulators = Simulators;
 			runner.Initialize ();