2016-05-26 16:06:52 +03:00
using System ;
using System.Collections.Generic ;
using System.Diagnostics ;
using System.IO ;
using System.Linq ;
using System.Runtime.InteropServices ;
using System.Text ;
using System.Threading ;
2016-06-06 13:48:53 +03:00
using System.Threading.Tasks ;
2016-05-26 16:06:52 +03:00
using System.Xml ;
namespace xharness
{
public class AppRunner
{
public Harness Harness ;
public string ProjectFile ;
2016-06-06 13:48:53 +03:00
public TestExecutingResult Result { get ; private set ; }
2016-05-26 16:06:52 +03:00
string appName ;
string appPath ;
string launchAppPath ;
string bundle_identifier ;
string platform ;
bool isSimulator ;
string device_name ;
string companion_device_name ;
2016-06-06 13:48:53 +03:00
// For watch apps we end up with 2 simulators, the watch simulator (the main one), and the iphone simulator (the companion one).
SimDevice [ ] simulators ;
SimDevice simulator { get { return simulators [ 0 ] ; } }
SimDevice companion_simulator { get { return simulators . Length = = 2 ? simulators [ 1 ] : null ; } }
string target ;
public string Target {
get { return target ? ? Harness . Target ; }
set { target = value ; }
2016-05-26 16:06:52 +03:00
}
2016-06-06 13:48:53 +03:00
string log_directory ;
public string LogDirectory {
get { return log_directory ? ? Harness . LogDirectory ; }
set { log_directory = value ; }
2016-05-26 16:06:52 +03:00
}
2016-06-17 18:21:18 +03:00
Log main_log ;
public Logs Logs = new Logs ( ) ;
public Log MainLog {
get { return main_log ; }
set { main_log = value ; }
}
2016-05-26 16:06:52 +03:00
2016-06-06 13:48:53 +03:00
public SimDevice [ ] Simulators {
get { return simulators ; }
set { simulators = value ; }
}
2016-05-26 16:06:52 +03:00
2016-06-17 18:21:18 +03:00
public string BundleIdentifier {
get {
return bundle_identifier ;
2016-05-26 16:06:52 +03:00
}
}
2016-06-17 18:21:18 +03:00
string mode ;
2016-05-26 16:06:52 +03:00
void FindSimulator ( )
{
2016-06-06 13:48:53 +03:00
if ( simulators ! = null )
return ;
2016-05-26 16:06:52 +03:00
string simulator_devicetype ;
string simulator_runtime ;
2016-06-06 13:48:53 +03:00
switch ( Target ) {
2016-05-26 16:06:52 +03:00
case "ios-simulator-32" :
simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.iPhone-5" ;
simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin . SdkVersions . iOS . Replace ( '.' , '-' ) ;
break ;
case "ios-simulator-64" :
simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.iPhone-5s" ;
simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin . SdkVersions . iOS . Replace ( '.' , '-' ) ;
break ;
case "ios-simulator" :
2016-06-16 09:39:16 +03:00
simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.iPhone-5" ;
2016-05-26 16:06:52 +03:00
simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin . SdkVersions . iOS . Replace ( '.' , '-' ) ;
break ;
case "tvos-simulator" :
simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.Apple-TV-1080p" ;
simulator_runtime = "com.apple.CoreSimulator.SimRuntime.tvOS-" + Xamarin . SdkVersions . TVOS . Replace ( '.' , '-' ) ;
break ;
case "watchos-simulator" :
simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.Apple-Watch-38mm" ;
simulator_runtime = "com.apple.CoreSimulator.SimRuntime.watchOS-" + Xamarin . SdkVersions . WatchOS . Replace ( '.' , '-' ) ;
break ;
default :
throw new Exception ( string . Format ( "Unknown simulator target: {0}" , Harness . Target ) ) ;
}
2016-06-06 13:48:53 +03:00
2016-06-16 04:57:47 +03:00
var sims = new Simulators ( ) {
Harness = Harness ,
} ;
2016-06-06 13:48:53 +03:00
Task . Run ( async ( ) = >
{
2016-06-17 18:21:18 +03:00
await sims . LoadAsync ( Logs . CreateStream ( LogDirectory , "simulator-list.log" , "Simulator list" ) ) ;
2016-06-06 13:48:53 +03:00
} ) . Wait ( ) ;
var devices = sims . AvailableDevices . Where ( ( SimDevice v ) = > v . SimRuntime = = simulator_runtime & & v . SimDeviceType = = simulator_devicetype ) ;
SimDevice candidate = null ;
simulators = null ;
foreach ( var device in devices ) {
var data = device ;
var secondaryData = ( SimDevice ) null ;
var nodeCompanions = sims . AvailableDevicePairs . Where ( ( SimDevicePair v ) = > v . Companion = = device . UDID ) ;
var nodeGizmos = sims . AvailableDevicePairs . Where ( ( SimDevicePair v ) = > v . Gizmo = = device . UDID ) ;
if ( nodeCompanions . Any ( ) ) {
var gizmo_udid = nodeCompanions . First ( ) . Gizmo ;
var node = sims . AvailableDevices . Where ( ( SimDevice v ) = > v . UDID = = gizmo_udid ) ;
secondaryData = node . First ( ) ;
} else if ( nodeGizmos . Any ( ) ) {
var companion_udid = nodeGizmos . First ( ) . Companion ;
var node = sims . AvailableDevices . Where ( ( SimDevice v ) = > v . UDID = = companion_udid ) ;
secondaryData = node . First ( ) ;
}
if ( secondaryData ! = null ) {
simulators = new SimDevice [ ] { data , secondaryData } ;
break ;
} else {
candidate = data ;
2016-05-26 16:06:52 +03:00
}
}
2016-06-24 11:03:36 +03:00
if ( simulators = = null ) {
if ( candidate = = null )
throw new Exception ( $"Could not find simulator for runtime={simulator_runtime} and device type={simulator_devicetype}." ) ;
2016-06-06 13:48:53 +03:00
simulators = new SimDevice [ ] { candidate } ;
2016-06-24 11:03:36 +03:00
}
2016-05-26 16:06:52 +03:00
if ( simulators = = null )
throw new Exception ( "Could not find simulator" ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Found simulator: {0} {1}" , simulators [ 0 ] . Name , simulators [ 0 ] . UDID ) ;
2016-05-26 16:06:52 +03:00
if ( simulators . Length > 1 )
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Found companion simulator: {0} {1}" , simulators [ 1 ] . Name , simulators [ 1 ] . UDID ) ;
2016-05-26 16:06:52 +03:00
}
void FindDevice ( )
{
if ( device_name ! = null )
return ;
device_name = Environment . GetEnvironmentVariable ( "DEVICE_NAME" ) ;
if ( ! string . IsNullOrEmpty ( device_name ) )
return ;
2016-06-17 18:21:18 +03:00
var devs = new Devices ( ) {
Harness = Harness ,
} ;
2016-06-06 13:48:53 +03:00
Task . Run ( async ( ) = >
{
2016-06-17 18:21:18 +03:00
await devs . LoadAsync ( main_log ) ;
2016-06-06 13:48:53 +03:00
} ) . Wait ( ) ;
2016-05-26 16:06:52 +03:00
2016-06-06 13:48:53 +03:00
string [ ] deviceClasses ;
switch ( mode ) {
case "ios" :
deviceClasses = new string [ ] { "iPhone" , "iPad" } ;
break ;
case "watchos" :
deviceClasses = new string [ ] { "Watch" } ;
break ;
case "tvos" :
deviceClasses = new string [ ] { "AppleTV" } ; // Untested
break ;
default :
throw new Exception ( $"unknown mode: {mode}" ) ;
}
2016-05-26 16:06:52 +03:00
2016-06-06 13:48:53 +03:00
var selected = devs . ConnectedDevices . Where ( ( v ) = > deviceClasses . Contains ( v . DeviceClass ) ) ;
Device selected_data ;
if ( selected . Count ( ) = = 0 ) {
throw new Exception ( $"Could not find any applicable devices with device class(es): {string.Join (" , ", deviceClasses)}" ) ;
} else if ( selected . Count ( ) > 1 ) {
selected_data = selected . First ( ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Found {0} devices for device class(es) {1}: {2}. Selected: '{3}'" , selected . Count ( ) , string . Join ( ", " , deviceClasses ) , string . Join ( ", " , selected . Select ( ( v ) = > v . Name ) . ToArray ( ) ) , selected_data . Name ) ;
2016-06-06 13:48:53 +03:00
} else {
selected_data = selected . First ( ) ;
}
device_name = selected_data . Name ;
if ( mode = = "watchos" ) {
var companion = devs . ConnectedDevices . Where ( ( v ) = > v . DeviceIdentifier = = selected_data . CompanionIdentifier ) ;
if ( companion . Count ( ) = = 0 )
throw new Exception ( $"Could not find the companion device for '{selected_data.Name}'" ) ;
else if ( companion . Count ( ) > 1 )
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Found {0} companion devices for {1}?!?" , companion . Count ( ) , selected_data . Name ) ;
2016-06-06 13:48:53 +03:00
companion_device_name = companion . First ( ) . Name ;
}
}
2016-06-07 19:49:20 +03:00
bool initialized ;
public void Initialize ( )
2016-05-26 16:06:52 +03:00
{
2016-06-07 19:49:20 +03:00
if ( initialized )
return ;
initialized = true ;
2016-05-26 16:06:52 +03:00
var csproj = new XmlDocument ( ) ;
csproj . LoadWithoutNetworkAccess ( ProjectFile ) ;
appName = csproj . GetAssemblyName ( ) ;
var info_plist_path = csproj . GetInfoPListInclude ( ) ;
var info_plist = new XmlDocument ( ) ;
info_plist . LoadWithoutNetworkAccess ( Path . Combine ( Path . GetDirectoryName ( ProjectFile ) , info_plist_path ) ) ;
bundle_identifier = info_plist . GetCFBundleIdentifier ( ) ;
2016-06-06 13:48:53 +03:00
switch ( Target ) {
2016-05-26 16:06:52 +03:00
case "ios-simulator-32" :
mode = "sim32" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
case "ios-simulator-64" :
mode = "sim64" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
case "ios-simulator" :
mode = "classic" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
case "ios-device" :
mode = "ios" ;
platform = "iPhone" ;
isSimulator = false ;
break ;
case "tvos-simulator" :
mode = "tvos" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
case "tvos-device" :
mode = "tvos" ;
platform = "iPhone" ;
isSimulator = false ;
break ;
case "watchos-simulator" :
mode = "watchos" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
case "watchos-device" :
mode = "watchos" ;
platform = "iPhone" ;
isSimulator = false ;
break ;
default :
throw new Exception ( string . Format ( "Unknown target: {0}" , Harness . Target ) ) ;
}
appPath = Path . Combine ( Path . GetDirectoryName ( ProjectFile ) , csproj . GetOutputPath ( platform , Harness . Configuration ) . Replace ( '\\' , '/' ) , appName + ".app" ) ;
if ( ! Directory . Exists ( appPath ) )
throw new Exception ( string . Format ( "The app directory {0} does not exist. This is probably a bug in the test harness." , appPath ) ) ;
if ( mode = = "watchos" ) {
launchAppPath = Directory . GetDirectories ( Path . Combine ( appPath , "Watch" ) , "*.app" ) [ 0 ] ;
} else {
launchAppPath = appPath ;
}
}
2016-06-17 18:21:18 +03:00
public int Install ( Log log )
2016-05-26 16:06:52 +03:00
{
Initialize ( ) ;
if ( isSimulator ) {
// We reset the simulator when running, so a separate install step does not make much sense.
throw new Exception ( "Installing to a simulator is not supported." ) ;
}
FindDevice ( ) ;
var args = new StringBuilder ( ) ;
if ( ! string . IsNullOrEmpty ( Harness . XcodeRoot ) )
args . Append ( " --sdkroot " ) . Append ( Harness . XcodeRoot ) ;
for ( int i = - 1 ; i < Harness . Verbosity ; i + + )
args . Append ( " -v " ) ;
args . Append ( " --installdev" ) ;
args . AppendFormat ( " \"{0}\" " , appPath ) ;
AddDeviceName ( args , companion_device_name ? ? device_name ) ;
if ( mode = = "watchos" )
args . Append ( " --device ios,watchos" ) ;
2016-06-17 18:21:18 +03:00
var rv = ProcessHelper . ExecuteCommandAsync ( Harness . MlaunchPath , args . ToString ( ) , log , TimeSpan . FromHours ( 1 ) ) . Result ;
return rv . Succeeded ? 0 : 1 ;
2016-05-26 16:06:52 +03:00
}
2016-06-17 18:21:18 +03:00
bool ensure_clean_simulator_state = true ;
public bool EnsureCleanSimulatorState {
2016-05-26 16:06:52 +03:00
get {
2016-06-17 18:21:18 +03:00
return ensure_clean_simulator_state | | ! string . IsNullOrEmpty ( Environment . GetEnvironmentVariable ( "SKIP_SIMULATOR_SETUP" ) ) ;
2016-06-06 13:48:53 +03:00
}
set {
2016-06-17 18:21:18 +03:00
ensure_clean_simulator_state = value ;
2016-06-06 13:48:53 +03:00
}
}
2016-06-17 18:21:18 +03:00
public async Task < int > RunAsync ( )
2016-05-26 16:06:52 +03:00
{
HashSet < string > start_crashes = null ;
2016-06-17 18:21:18 +03:00
LogStream device_system_log = null ;
LogStream listener_log = null ;
Log run_log = main_log ;
2016-05-26 16:06:52 +03:00
Initialize ( ) ;
var args = new StringBuilder ( ) ;
if ( ! string . IsNullOrEmpty ( Harness . XcodeRoot ) )
args . Append ( " --sdkroot " ) . Append ( Harness . XcodeRoot ) ;
for ( int i = - 1 ; i < Harness . Verbosity ; i + + )
args . Append ( " -v " ) ;
args . Append ( " -argument=-connection-mode -argument=none" ) ; // This will prevent the app from trying to connect to any IDEs
args . Append ( " -argument=-app-arg:-autostart" ) ;
args . Append ( " -setenv=NUNIT_AUTOSTART=true" ) ;
args . Append ( " -argument=-app-arg:-autoexit" ) ;
args . Append ( " -setenv=NUNIT_AUTOEXIT=true" ) ;
args . Append ( " -argument=-app-arg:-enablenetwork" ) ;
args . Append ( " -setenv=NUNIT_ENABLE_NETWORK=true" ) ;
if ( isSimulator ) {
args . Append ( " -argument=-app-arg:-hostname:127.0.0.1" ) ;
args . Append ( " -setenv=NUNIT_HOSTNAME=127.0.0.1" ) ;
} else {
var ips = new StringBuilder ( ) ;
var ipAddresses = System . Net . Dns . GetHostEntry ( System . Net . Dns . GetHostName ( ) ) . AddressList ;
for ( int i = 0 ; i < ipAddresses . Length ; i + + ) {
if ( i > 0 )
ips . Append ( ',' ) ;
ips . Append ( ipAddresses [ i ] . ToString ( ) ) ;
}
args . AppendFormat ( " -argument=-app-arg:-hostname:{0}" , ips . ToString ( ) ) ;
args . AppendFormat ( " -setenv=NUNIT_HOSTNAME={0}" , ips . ToString ( ) ) ;
}
var transport = mode = = "watchos" ? "HTTP" : "TCP" ;
args . AppendFormat ( " -argument=-app-arg:-transport:{0}" , transport ) ;
args . AppendFormat ( " -setenv=NUNIT_TRANSPORT={0}" , transport ) ;
SimpleListener listener ;
switch ( transport ) {
case "HTTP" :
listener = new SimpleHttpListener ( ) ;
break ;
case "TCP" :
listener = new SimpleTcpListener ( ) ;
break ;
default :
throw new NotImplementedException ( ) ;
}
2016-06-17 18:21:18 +03:00
listener_log = Logs . CreateStream ( LogDirectory , string . Format ( "test-{0:yyyyMMdd_HHmmss}.log" , DateTime . Now ) , "Test log" ) ;
listener . Log = listener_log ;
2016-05-26 16:06:52 +03:00
listener . AutoExit = true ;
listener . Address = System . Net . IPAddress . Any ;
listener . Initialize ( ) ;
args . AppendFormat ( " -argument=-app-arg:-hostport:{0}" , listener . Port ) ;
args . AppendFormat ( " -setenv=NUNIT_HOSTPORT={0}" , listener . Port ) ;
bool? success = null ;
bool timed_out = false ;
if ( isSimulator ) {
2016-06-17 18:21:18 +03:00
FindSimulator ( ) ;
2016-06-16 10:07:47 +03:00
var systemLogs = new List < CaptureLog > ( ) ;
foreach ( var sim in simulators ) {
// Upload the system log
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "System log for the '{1}' simulator is: {0}" , sim . SystemLog , sim . Name ) ;
bool isCompanion = sim ! = simulator ;
var log = new CaptureLog ( sim . SystemLog ) {
Path = Path . Combine ( LogDirectory , sim . UDID + ".log" ) ,
Description = isCompanion ? "System log (companion)" : "System log" ,
} ;
log . StartCapture ( ) ;
Logs . Add ( log ) ;
systemLogs . Add ( log ) ;
Harness . LogWrench ( "@MonkeyWrench: AddFile: {0}" , log . Path ) ;
2016-06-16 10:07:47 +03:00
}
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "*** Executing {0}/{1} in the simulator ***" , appName , mode ) ;
2016-06-16 10:07:47 +03:00
2016-06-17 18:21:18 +03:00
if ( EnsureCleanSimulatorState ) {
foreach ( var sim in simulators )
await sim . PrepareSimulatorAsync ( main_log , bundle_identifier ) ;
}
2016-05-26 16:06:52 +03:00
args . Append ( " --launchsim" ) ;
args . AppendFormat ( " \"{0}\" " , launchAppPath ) ;
2016-06-06 13:48:53 +03:00
args . Append ( " --device=:v2:udid=" ) . Append ( simulator . UDID ) . Append ( " " ) ;
2016-05-26 16:06:52 +03:00
2016-06-17 18:21:18 +03:00
start_crashes = await Harness . CreateCrashReportsSnapshotAsync ( main_log , true ) ;
2016-05-26 16:06:52 +03:00
listener . StartAsync ( ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Starting test run" ) ;
2016-05-26 16:06:52 +03:00
2016-06-17 18:21:18 +03:00
var cancellation_source = new CancellationTokenSource ( ) ;
2016-05-26 16:06:52 +03:00
ThreadPool . QueueUserWorkItem ( ( v ) = > {
if ( ! listener . WaitForConnection ( TimeSpan . FromMinutes ( Harness . LaunchTimeout ) ) ) {
2016-06-17 18:21:18 +03:00
cancellation_source . Cancel ( ) ;
main_log . WriteLine ( "Test launch timed out after {0} minute(s)." , Harness . LaunchTimeout ) ;
2016-05-26 16:06:52 +03:00
} else {
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Test run started" ) ;
2016-05-26 16:06:52 +03:00
}
} ) ;
2016-06-17 18:21:18 +03:00
var result = await ProcessHelper . ExecuteCommandAsync ( Harness . MlaunchPath , args . ToString ( ) , run_log , TimeSpan . FromMinutes ( Harness . Timeout ) , cancellation_token : cancellation_source . Token ) ;
if ( result . TimedOut ) {
2016-05-26 16:06:52 +03:00
timed_out = true ;
2016-06-17 18:21:18 +03:00
success = false ;
main_log . WriteLine ( "Test run timed out after {0} minute(s)." , Harness . Timeout ) ;
} else if ( result . Succeeded ) {
main_log . WriteLine ( "Test run completed" ) ;
success = true ;
} else {
main_log . WriteLine ( "Test run failed" ) ;
success = false ;
}
2016-05-26 16:06:52 +03:00
2016-06-17 18:21:18 +03:00
if ( ! success . Value ) {
2016-05-26 16:06:52 +03:00
// find pid
var pid = - 1 ;
2016-06-17 18:21:18 +03:00
using ( var reader = run_log . GetReader ( ) ) {
while ( ! reader . EndOfStream ) {
var line = reader . ReadLine ( ) ;
if ( line . StartsWith ( "Application launched. PID = " , StringComparison . Ordinal ) ) {
var pidstr = line . Substring ( "Application launched. PID = " . Length ) ;
if ( ! int . TryParse ( pidstr , out pid ) )
main_log . WriteLine ( "Could not parse pid: {0}" , pidstr ) ;
} else if ( line . Contains ( "Xamarin.Hosting: Launched " ) & & line . Contains ( " with pid " ) ) {
var pidstr = line . Substring ( line . LastIndexOf ( ' ' ) ) ;
if ( ! int . TryParse ( pidstr , out pid ) )
main_log . WriteLine ( "Could not parse pid: {0}" , pidstr ) ;
}
2016-05-26 16:06:52 +03:00
}
}
if ( pid > 0 ) {
2016-06-17 18:21:18 +03:00
var launchTimedout = cancellation_source . IsCancellationRequested ;
await KillPidAsync ( main_log , pid , TimeSpan . FromSeconds ( 5 ) , TimeSpan . FromMinutes ( launchTimedout ? Harness . LaunchTimeout : Harness . Timeout ) , launchTimedout ? "Launch" : "Completion" ) ;
2016-05-26 16:06:52 +03:00
} else {
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Could not find pid in mtouch output." ) ;
2016-05-26 16:06:52 +03:00
}
}
listener . Cancel ( ) ;
// cleanup after us
2016-06-17 18:21:18 +03:00
if ( EnsureCleanSimulatorState )
await SimDevice . KillEverythingAsync ( main_log ) ;
2016-06-16 10:07:47 +03:00
foreach ( var log in systemLogs )
log . StopCapture ( ) ;
2016-05-26 16:06:52 +03:00
} else {
FindDevice ( ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "*** Executing {0}/{1} on device ***" , appName , mode ) ;
2016-05-26 16:06:52 +03:00
args . Append ( " --launchdev" ) ;
args . AppendFormat ( " \"{0}\" " , launchAppPath ) ;
AddDeviceName ( args ) ;
2016-06-17 18:21:18 +03:00
device_system_log = Logs . CreateStream ( LogDirectory , "device.log" , "Device log" ) ;
2016-05-26 16:06:52 +03:00
var logdev = new DeviceLogCapturer ( ) {
Harness = Harness ,
2016-06-17 18:21:18 +03:00
Log = device_system_log ,
2016-05-26 16:06:52 +03:00
DeviceName = device_name ,
} ;
logdev . StartCapture ( ) ;
2016-06-17 18:21:18 +03:00
start_crashes = await Harness . CreateCrashReportsSnapshotAsync ( main_log , false ) ;
2016-05-26 16:06:52 +03:00
listener . StartAsync ( ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Starting test run" ) ;
// This will not wait for app completion
await ProcessHelper . ExecuteCommandAsync ( Harness . MlaunchPath , args . ToString ( ) , main_log , TimeSpan . FromMinutes ( 1 ) ) ;
2016-05-26 16:06:52 +03:00
if ( listener . WaitForCompletion ( TimeSpan . FromMinutes ( Harness . Timeout ) ) ) {
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Test run completed" ) ;
2016-05-26 16:06:52 +03:00
} else {
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Test run did not complete in {0} minutes." , Harness . Timeout ) ;
2016-05-26 16:06:52 +03:00
listener . Cancel ( ) ;
success = false ;
timed_out = true ;
}
logdev . StopCapture ( ) ;
// Upload the system log
2016-06-17 18:21:18 +03:00
if ( File . Exists ( device_system_log . FullPath ) ) {
main_log . WriteLine ( "A capture of the device log is: {0}" , device_system_log . FullPath ) ;
2016-05-26 16:06:52 +03:00
if ( Harness . InWrench )
2016-06-17 18:21:18 +03:00
Harness . LogWrench ( "@MonkeyWrench: AddFile: {0}" , device_system_log . FullPath ) ;
2016-05-26 16:06:52 +03:00
}
}
listener . Dispose ( ) ;
// check the final status
var crashed = false ;
2016-06-17 18:21:18 +03:00
if ( File . Exists ( listener_log . FullPath ) ) {
Harness . LogWrench ( "@MonkeyWrench: AddFile: {0}" , listener_log . FullPath ) ;
string log ;
using ( var reader = listener_log . GetReader ( ) )
log = reader . ReadToEnd ( ) ;
2016-05-26 16:06:52 +03:00
if ( log . Contains ( "Tests run" ) ) {
var tests_run = string . Empty ;
2016-06-17 18:21:18 +03:00
var log_lines = log . Split ( '\n' ) ;
2016-05-26 16:06:52 +03:00
var failed = false ;
foreach ( var line in log_lines ) {
if ( line . Contains ( "Tests run:" ) ) {
Console . WriteLine ( line ) ;
tests_run = line . Replace ( "Tests run: " , "" ) ;
break ;
} else if ( line . Contains ( "FAIL" ) ) {
Console . WriteLine ( line ) ;
failed = true ;
}
}
if ( failed ) {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>" , mode , tests_run ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Test run failed" ) ;
success = false ;
2016-05-26 16:06:52 +03:00
} else {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>" , mode , tests_run ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Test run succeeded" ) ;
2016-05-26 16:06:52 +03:00
success = true ;
}
} else if ( timed_out ) {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>" , mode ) ;
} else {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>" , mode ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Test run crashed" ) ;
2016-05-26 16:06:52 +03:00
crashed = true ;
}
} else if ( timed_out ) {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>" , mode ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Test run never launched" ) ;
2016-05-26 16:06:52 +03:00
} else {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>" , mode ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Test run crashed before it started (no log file produced)" ) ;
2016-05-26 16:06:52 +03:00
crashed = true ;
}
// Check for crash reports
var crash_report_search_done = false ;
var crash_report_search_timeout = 5 ;
var watch = new Stopwatch ( ) ;
watch . Start ( ) ;
do {
2016-06-17 18:21:18 +03:00
var end_crashes = await Harness . CreateCrashReportsSnapshotAsync ( main_log , isSimulator ) ;
2016-05-26 16:06:52 +03:00
end_crashes . ExceptWith ( start_crashes ) ;
if ( end_crashes . Count > 0 ) {
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Found {0} new crash report(s)" , end_crashes . Count ) ;
2016-06-06 13:48:53 +03:00
List < LogFile > crash_reports ;
if ( isSimulator ) {
crash_reports = new List < LogFile > ( end_crashes . Count ) ;
foreach ( var path in end_crashes ) {
2016-06-17 18:21:18 +03:00
var logPath = Path . Combine ( LogDirectory , Path . GetFileName ( path ) ) ;
File . Copy ( path , logPath , true ) ;
crash_reports . Add ( Logs . CreateFile ( "Crash report: " + Path . GetFileName ( path ) , logPath ) ) ;
2016-06-06 13:48:53 +03:00
}
} else {
2016-05-26 16:06:52 +03:00
// Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench
// (if we put them in /tmp, they'd never be deleted).
2016-06-06 13:48:53 +03:00
var downloaded_crash_reports = new List < LogFile > ( ) ;
2016-05-26 16:06:52 +03:00
foreach ( var file in end_crashes ) {
2016-06-17 18:21:18 +03:00
var crash_report_target = Logs . CreateFile ( "Crash report: " + Path . GetFileName ( file ) , Path . Combine ( LogDirectory , Path . GetFileName ( file ) ) ) ;
var result = await ProcessHelper . ExecuteCommandAsync ( Harness . MlaunchPath , "--download-crash-report=" + file + " --download-crash-report-to=" + crash_report_target . Path + " --sdkroot " + Harness . XcodeRoot , main_log , TimeSpan . FromMinutes ( 1 ) ) ;
if ( result . Succeeded ) {
main_log . WriteLine ( "Downloaded crash report {0} to {1}" , file , crash_report_target . Path ) ;
crash_report_target = await Harness . SymbolicateCrashReportAsync ( main_log , crash_report_target ) ;
Logs . Add ( crash_report_target ) ;
2016-05-26 16:06:52 +03:00
downloaded_crash_reports . Add ( crash_report_target ) ;
} else {
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "Could not download crash report {0}" , file ) ;
2016-05-26 16:06:52 +03:00
}
}
2016-06-06 13:48:53 +03:00
crash_reports = downloaded_crash_reports ;
2016-05-26 16:06:52 +03:00
}
2016-06-06 13:48:53 +03:00
foreach ( var cp in crash_reports ) {
Harness . LogWrench ( "@MonkeyWrench: AddFile: {0}" , cp . Path ) ;
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( " {0}" , cp . Path ) ;
2016-05-26 16:06:52 +03:00
}
crash_report_search_done = true ;
} else if ( ! crashed & & ! timed_out ) {
crash_report_search_done = true ;
} else {
if ( watch . Elapsed . TotalSeconds > crash_report_search_timeout ) {
crash_report_search_done = true ;
} else {
2016-06-17 18:21:18 +03:00
main_log . WriteLine ( "No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})" , ( int ) ( crash_report_search_timeout - watch . Elapsed . TotalSeconds ) ) ;
2016-05-26 16:06:52 +03:00
Thread . Sleep ( TimeSpan . FromSeconds ( 1 ) ) ;
}
}
} while ( ! crash_report_search_done ) ;
if ( ! success . HasValue )
success = false ;
2016-06-16 10:07:47 +03:00
2016-06-17 18:21:18 +03:00
if ( timed_out ) {
2016-06-06 13:48:53 +03:00
Result = TestExecutingResult . TimedOut ;
} else if ( crashed ) {
Result = TestExecutingResult . Crashed ;
2016-06-17 18:21:18 +03:00
} else if ( success . Value ) {
Result = TestExecutingResult . Succeeded ;
2016-06-06 13:48:53 +03:00
} else {
Result = TestExecutingResult . Failed ;
}
2016-05-26 16:06:52 +03:00
return success . Value ? 0 : 1 ;
}
public void AddDeviceName ( StringBuilder args )
{
AddDeviceName ( args , device_name ) ;
}
public static void AddDeviceName ( StringBuilder args , string device_name )
{
if ( ! string . IsNullOrEmpty ( device_name ) ) {
args . Append ( " --devname " ) ;
args . Append ( Harness . Quote ( device_name ) ) ;
}
}
[DllImport ("/usr/lib/libc.dylib")]
static extern void kill ( int pid , int sig ) ;
2016-06-17 18:21:18 +03:00
async Task KillPidAsync ( Log log , int pid , TimeSpan kill_separation , TimeSpan timeout , string type )
2016-05-26 16:06:52 +03:00
{
2016-06-17 18:21:18 +03:00
log . WriteLine ( "{2} timeout ({1} s) reached, will now send SIGQUIT to the app (PID: {0})" , pid , timeout . TotalSeconds , type ) ;
2016-05-26 16:06:52 +03:00
kill ( pid , 3 /* SIGQUIT */ ) ; // print managed stack traces.
2016-06-17 18:21:18 +03:00
if ( await ProcessHelper . PollForExitAsync ( pid , kill_separation /* wait for at most 5 seconds to see if something happens */ ) )
2016-06-06 13:48:53 +03:00
return ;
2016-06-17 18:21:18 +03:00
log . WriteLine ( "{2} timeout ({1} s) reached, will now send SIGABRT to the app (PID: {0})" , pid , timeout . TotalSeconds , type ) ;
kill ( pid , 6 /* SIGABRT */ ) ; // print native stack traces.
if ( await ProcessHelper . PollForExitAsync ( pid , kill_separation /* wait another 5 seconds */ ) )
2016-06-07 19:49:20 +03:00
return ;
2016-06-17 18:21:18 +03:00
log . WriteLine ( "{2} timeout ({1} s) reached, will now send SIGKILL to the app (PID: {0})" , pid , timeout . TotalSeconds , type ) ;
kill ( pid , 9 /* SIGKILL */ ) ; // terminate unconditionally.
2016-05-26 16:06:52 +03:00
}
}
}