2019-05-11 13:03:38 +03:00
using System ;
2016-05-26 16:06:52 +03:00
using System.Collections.Generic ;
using System.Diagnostics ;
using System.IO ;
using System.Linq ;
using System.Runtime.InteropServices ;
using System.Text ;
using System.Threading ;
2016-06-06 13:48:53 +03:00
using System.Threading.Tasks ;
2016-05-26 16:06:52 +03:00
using System.Xml ;
2020-03-10 22:10:37 +03:00
using Xamarin ;
using Xharness.Hardware ;
2020-03-06 17:11:33 +03:00
using Xharness.Execution ;
2020-03-06 14:00:34 +03:00
using Xharness.Jenkins.TestTasks ;
2020-03-05 14:41:52 +03:00
using Xharness.Listeners ;
2020-03-03 21:59:44 +03:00
using Xharness.Logging ;
2020-03-10 15:12:33 +03:00
using Xharness.Utilities ;
2016-05-26 16:06:52 +03:00
2020-03-10 19:44:18 +03:00
namespace Xharness {
2016-11-16 17:23:11 +03:00
public enum AppRunnerTarget
{
None ,
Simulator_iOS ,
Simulator_iOS32 ,
Simulator_iOS64 ,
Simulator_tvOS ,
Simulator_watchOS ,
Device_iOS ,
Device_tvOS ,
Device_watchOS ,
}
2017-01-04 21:45:26 +03:00
public enum Extension
{
WatchKit2 ,
TodayExtension ,
}
2020-02-14 00:51:14 +03:00
class AppRunner
2016-05-26 16:06:52 +03:00
{
2020-03-10 19:44:18 +03:00
string appPath ;
string launchAppPath ;
string platform ;
Extension ? extension ;
bool isSimulator ;
string configuration ;
string mode ;
bool initialized ;
2016-05-26 16:06:52 +03:00
public Harness Harness ;
public string ProjectFile ;
2017-01-04 21:45:26 +03:00
public string AppPath ;
2020-02-11 22:03:16 +03:00
public string Variation ;
2020-02-14 00:51:14 +03:00
public BuildToolTask BuildTask ;
2020-03-05 14:41:52 +03:00
public ISimpleListenerFactory ListenerFactory = new SimpleListenerFactory ( ) ;
2016-05-26 16:06:52 +03:00
2016-06-06 13:48:53 +03:00
public TestExecutingResult Result { get ; private set ; }
2020-03-10 19:44:18 +03:00
2017-02-08 13:44:20 +03:00
public string FailureMessage { get ; private set ; }
2016-05-26 16:06:52 +03:00
2020-03-10 19:44:18 +03:00
public string DeviceName { get ; set ; }
2016-05-26 16:06:52 +03:00
2020-03-10 19:44:18 +03:00
public string CompanionDeviceName { get ; set ; }
2016-05-26 16:06:52 +03:00
2020-03-10 19:44:18 +03:00
public bool IsExtension = > extension . HasValue ;
2017-01-04 21:45:26 +03:00
2020-03-10 19:44:18 +03:00
public string AppName { get ; private set ; }
2017-01-04 21:45:26 +03:00
2020-03-10 19:44:18 +03:00
public double TimeoutMultiplier { get ; set ; } = 1 ;
2017-01-04 21:45:26 +03:00
2020-03-10 19:44:18 +03:00
ILogs logs ;
public ILogs Logs = > logs ? ? ( logs = new Logs ( LogDirectory ) ) ;
2017-01-04 21:45:26 +03:00
2020-03-10 19:44:18 +03:00
public ILog MainLog { get ; set ; }
2020-02-15 01:28:00 +03:00
2020-03-10 22:10:37 +03:00
public ISimulatorDevice [ ] Simulators { get ; set ; }
ISimulatorDevice simulator = > Simulators [ 0 ] ;
2019-05-30 18:40:56 +03:00
2020-03-10 19:44:18 +03:00
public string BundleIdentifier { get ; private set ; }
public IProcessManager ProcessManager { get ; set ; } = new ProcessManager ( ) ;
2016-06-06 13:48:53 +03:00
2016-11-16 17:23:11 +03:00
AppRunnerTarget target ;
public AppRunnerTarget Target {
2020-03-10 19:44:18 +03:00
get = > target = = AppRunnerTarget . None ? Harness . Target : target ;
set = > target = value ;
2016-05-26 16:06:52 +03:00
}
2016-06-06 13:48:53 +03:00
string log_directory ;
public string LogDirectory {
2020-03-10 19:44:18 +03:00
get = > log_directory ? ? Harness . LogDirectory ;
set = > log_directory = value ;
2016-05-26 16:06:52 +03:00
}
2020-03-10 19:44:18 +03:00
bool ensure_clean_simulator_state = true ;
public bool EnsureCleanSimulatorState {
get = > ensure_clean_simulator_state & & string . IsNullOrEmpty ( Environment . GetEnvironmentVariable ( "SKIP_SIMULATOR_SETUP" ) ) ;
set = > ensure_clean_simulator_state = value ;
2016-06-06 13:48:53 +03:00
}
2016-05-26 16:06:52 +03:00
2020-03-10 19:44:18 +03:00
public string Configuration {
get = > configuration ? ? Harness . Configuration ;
set = > configuration = value ;
2016-05-26 16:06:52 +03:00
}
2016-11-16 17:23:11 +03:00
async Task < bool > FindSimulatorAsync ( )
2016-05-26 16:06:52 +03:00
{
2020-03-10 19:44:18 +03:00
if ( Simulators ! = null )
2016-11-16 17:23:11 +03:00
return true ;
2016-06-06 13:48:53 +03:00
2016-06-16 04:57:47 +03:00
var sims = new Simulators ( ) {
Harness = Harness ,
} ;
2018-10-10 09:05:26 +03:00
await sims . LoadAsync ( Logs . Create ( $"simulator-list-{Harness.Timestamp}.log" , "Simulator list" ) ) ;
2020-03-10 19:44:18 +03:00
Simulators = await sims . FindAsync ( Target , MainLog ) ;
2016-05-26 16:06:52 +03:00
2020-03-10 19:44:18 +03:00
return Simulators ! = null ;
2016-05-26 16:06:52 +03:00
}
void FindDevice ( )
{
2020-03-10 19:44:18 +03:00
if ( DeviceName ! = null )
2016-05-26 16:06:52 +03:00
return ;
2020-03-10 19:44:18 +03:00
DeviceName = Environment . GetEnvironmentVariable ( "DEVICE_NAME" ) ;
if ( ! string . IsNullOrEmpty ( DeviceName ) )
2016-05-26 16:06:52 +03:00
return ;
2016-06-17 18:21:18 +03:00
var devs = new Devices ( ) {
Harness = Harness ,
} ;
2016-06-06 13:48:53 +03:00
Task . Run ( async ( ) = >
{
2020-03-10 19:44:18 +03:00
await devs . LoadAsync ( MainLog ) ;
2016-06-06 13:48:53 +03:00
} ) . Wait ( ) ;
2016-05-26 16:06:52 +03:00
2016-06-06 13:48:53 +03:00
string [ ] deviceClasses ;
switch ( mode ) {
case "ios" :
2018-01-30 19:11:04 +03:00
deviceClasses = new string [ ] { "iPhone" , "iPad" , "iPod" } ;
2016-06-06 13:48:53 +03:00
break ;
case "watchos" :
deviceClasses = new string [ ] { "Watch" } ;
break ;
case "tvos" :
deviceClasses = new string [ ] { "AppleTV" } ; // Untested
break ;
default :
throw new Exception ( $"unknown mode: {mode}" ) ;
}
2016-05-26 16:06:52 +03:00
2018-02-12 16:43:16 +03:00
var selected = devs . ConnectedDevices . Where ( ( v ) = > deviceClasses . Contains ( v . DeviceClass ) & & v . IsUsableForDebugging ! = false ) ;
2020-03-10 22:10:37 +03:00
IHardwareDevice selected_data ;
2016-06-06 13:48:53 +03:00
if ( selected . Count ( ) = = 0 ) {
throw new Exception ( $"Could not find any applicable devices with device class(es): {string.Join (" , ", deviceClasses)}" ) ;
} else if ( selected . Count ( ) > 1 ) {
2016-12-22 20:55:25 +03:00
selected_data = selected
. OrderBy ( ( dev ) = >
{
2020-03-10 19:44:18 +03:00
if ( Version . TryParse ( dev . ProductVersion , out Version v ) )
2016-12-22 20:55:25 +03:00
return v ;
return new Version ( ) ;
} )
. First ( ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Found {0} devices for device class(es) '{1}': '{2}'. Selected: '{3}' (because it has the lowest version)." , selected . Count ( ) , string . Join ( "', '" , deviceClasses ) , string . Join ( "', '" , selected . Select ( ( v ) = > v . Name ) . ToArray ( ) ) , selected_data . Name ) ;
2016-06-06 13:48:53 +03:00
} else {
selected_data = selected . First ( ) ;
}
2020-03-10 19:44:18 +03:00
DeviceName = selected_data . Name ;
2016-06-06 13:48:53 +03:00
2017-01-04 21:45:26 +03:00
if ( mode = = "watchos" )
2020-03-10 19:44:18 +03:00
CompanionDeviceName = devs . FindCompanionDevice ( MainLog , selected_data ) . Name ;
2016-06-06 13:48:53 +03:00
}
2016-06-07 19:49:20 +03:00
public void Initialize ( )
2016-05-26 16:06:52 +03:00
{
2016-06-07 19:49:20 +03:00
if ( initialized )
return ;
initialized = true ;
2016-05-26 16:06:52 +03:00
var csproj = new XmlDocument ( ) ;
csproj . LoadWithoutNetworkAccess ( ProjectFile ) ;
2020-03-10 19:44:18 +03:00
AppName = csproj . GetAssemblyName ( ) ;
2016-05-26 16:06:52 +03:00
var info_plist_path = csproj . GetInfoPListInclude ( ) ;
var info_plist = new XmlDocument ( ) ;
2017-01-04 21:45:26 +03:00
info_plist . LoadWithoutNetworkAccess ( Path . Combine ( Path . GetDirectoryName ( ProjectFile ) , info_plist_path . Replace ( '\\' , '/' ) ) ) ;
2020-03-10 19:44:18 +03:00
BundleIdentifier = info_plist . GetCFBundleIdentifier ( ) ;
2016-05-26 16:06:52 +03:00
2017-01-04 21:45:26 +03:00
var extensionPointIdentifier = info_plist . GetNSExtensionPointIdentifier ( ) ;
if ( ! string . IsNullOrEmpty ( extensionPointIdentifier ) )
extension = extensionPointIdentifier . ParseFromNSExtensionPointIdentifier ( ) ;
2016-06-06 13:48:53 +03:00
switch ( Target ) {
2016-11-16 17:23:11 +03:00
case AppRunnerTarget . Simulator_iOS32 :
2016-05-26 16:06:52 +03:00
mode = "sim32" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
2016-11-16 17:23:11 +03:00
case AppRunnerTarget . Simulator_iOS64 :
2016-05-26 16:06:52 +03:00
mode = "sim64" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
2016-11-16 17:23:11 +03:00
case AppRunnerTarget . Simulator_iOS :
2016-05-26 16:06:52 +03:00
mode = "classic" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
2016-11-16 17:23:11 +03:00
case AppRunnerTarget . Device_iOS :
2016-05-26 16:06:52 +03:00
mode = "ios" ;
platform = "iPhone" ;
isSimulator = false ;
break ;
2016-11-16 17:23:11 +03:00
case AppRunnerTarget . Simulator_tvOS :
2016-05-26 16:06:52 +03:00
mode = "tvos" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
2016-11-16 17:23:11 +03:00
case AppRunnerTarget . Device_tvOS :
2016-05-26 16:06:52 +03:00
mode = "tvos" ;
platform = "iPhone" ;
isSimulator = false ;
break ;
2016-11-16 17:23:11 +03:00
case AppRunnerTarget . Simulator_watchOS :
2016-05-26 16:06:52 +03:00
mode = "watchos" ;
platform = "iPhoneSimulator" ;
isSimulator = true ;
break ;
2016-11-16 17:23:11 +03:00
case AppRunnerTarget . Device_watchOS :
2016-05-26 16:06:52 +03:00
mode = "watchos" ;
platform = "iPhone" ;
isSimulator = false ;
break ;
default :
throw new Exception ( string . Format ( "Unknown target: {0}" , Harness . Target ) ) ;
}
2020-03-10 19:44:18 +03:00
appPath = Path . Combine ( Path . GetDirectoryName ( ProjectFile ) , csproj . GetOutputPath ( platform , Configuration ) . Replace ( '\\' , '/' ) , AppName + ( IsExtension ? ".appex" : ".app" ) ) ;
2016-05-26 16:06:52 +03:00
if ( ! Directory . Exists ( appPath ) )
throw new Exception ( string . Format ( "The app directory {0} does not exist. This is probably a bug in the test harness." , appPath ) ) ;
if ( mode = = "watchos" ) {
launchAppPath = Directory . GetDirectories ( Path . Combine ( appPath , "Watch" ) , "*.app" ) [ 0 ] ;
} else {
launchAppPath = appPath ;
}
}
2019-02-20 08:33:30 +03:00
public async Task < ProcessExecutionResult > InstallAsync ( CancellationToken cancellation_token )
2016-05-26 16:06:52 +03:00
{
Initialize ( ) ;
if ( isSimulator ) {
// We reset the simulator when running, so a separate install step does not make much sense.
throw new Exception ( "Installing to a simulator is not supported." ) ;
}
FindDevice ( ) ;
2019-10-14 17:18:46 +03:00
var args = new List < string > ( ) ;
if ( ! string . IsNullOrEmpty ( Harness . XcodeRoot ) ) {
args . Add ( "--sdkroot" ) ;
args . Add ( Harness . XcodeRoot ) ;
}
2016-05-26 16:06:52 +03:00
for ( int i = - 1 ; i < Harness . Verbosity ; i + + )
2019-10-14 17:18:46 +03:00
args . Add ( "-v" ) ;
2016-05-26 16:06:52 +03:00
2019-10-14 17:18:46 +03:00
args . Add ( "--installdev" ) ;
args . Add ( appPath ) ;
2020-03-10 19:44:18 +03:00
AddDeviceName ( args , CompanionDeviceName ? ? DeviceName ) ;
2016-05-26 16:06:52 +03:00
2019-10-14 17:18:46 +03:00
if ( mode = = "watchos" ) {
args . Add ( "--device" ) ;
args . Add ( "ios,watchos" ) ;
}
2016-05-26 16:06:52 +03:00
2019-02-20 08:33:30 +03:00
var totalSize = Directory . GetFiles ( appPath , "*" , SearchOption . AllDirectories ) . Select ( ( v ) = > new FileInfo ( v ) . Length ) . Sum ( ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( $"Installing '{appPath}' to '{CompanionDeviceName ?? DeviceName}'. Size: {totalSize} bytes = {totalSize / 1024.0 / 1024.0:N2} MB" ) ;
2017-10-12 18:08:54 +03:00
2020-03-10 19:44:18 +03:00
return await ProcessManager . ExecuteCommandAsync ( Harness . MlaunchPath , args , MainLog , TimeSpan . FromHours ( 1 ) , cancellation_token : cancellation_token ) ;
2016-05-26 16:06:52 +03:00
}
2017-01-04 21:45:26 +03:00
public async Task < ProcessExecutionResult > UninstallAsync ( )
2017-01-02 10:58:46 +03:00
{
Initialize ( ) ;
if ( isSimulator )
throw new Exception ( "Uninstalling from a simulator is not supported." ) ;
FindDevice ( ) ;
2019-10-14 17:18:46 +03:00
var args = new List < string > ( ) ;
if ( ! string . IsNullOrEmpty ( Harness . XcodeRoot ) ) {
args . Add ( "--sdkroot" ) ;
args . Add ( Harness . XcodeRoot ) ;
}
2017-01-02 10:58:46 +03:00
for ( int i = - 1 ; i < Harness . Verbosity ; i + + )
2019-10-14 17:18:46 +03:00
args . Add ( "-v" ) ;
2017-01-02 10:58:46 +03:00
2019-10-14 17:18:46 +03:00
args . Add ( "--uninstalldevbundleid" ) ;
2020-03-10 19:44:18 +03:00
args . Add ( BundleIdentifier ) ;
AddDeviceName ( args , CompanionDeviceName ? ? DeviceName ) ;
2017-01-02 10:58:46 +03:00
2020-03-10 19:44:18 +03:00
return await ProcessManager . ExecuteCommandAsync ( Harness . MlaunchPath , args , MainLog , TimeSpan . FromMinutes ( 1 ) ) ;
2016-06-06 13:48:53 +03:00
}
2018-11-16 21:31:40 +03:00
2020-02-19 21:18:34 +03:00
( string resultLine , bool failed , bool crashed ) ParseResult ( string test_log_path , bool timed_out , out bool crashed )
2016-11-15 21:04:37 +03:00
{
2020-02-19 21:18:34 +03:00
crashed = false ;
if ( ! File . Exists ( test_log_path ) ) {
crashed = true ;
2019-05-23 11:04:37 +03:00
return ( null , false , true ) ; // if we do not have a log file, the test crashes
2020-02-19 21:18:34 +03:00
}
2017-06-26 18:34:32 +03:00
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
2019-05-23 11:04:37 +03:00
//
// On the other hand, the nunit and xunit do not have that data and have to be parsed.
2020-01-28 22:44:34 +03:00
//
// This if statement has a small trick, we found out that internet sharing in some of the bots (VSTS) does not work, in
// that case, we cannot do a TCP connection to xharness to get the log, this is a problem since if we did not get the xml
// from the TCP connection, we are going to fail when trying to read it and not parse it. Therefore, we are not only
// going to check if we are in CI, but also if the listener_log is valid.
2020-02-04 19:03:38 +03:00
var path = Path . ChangeExtension ( test_log_path , "xml" ) ;
2020-02-06 05:23:44 +03:00
XmlResultParser . CleanXml ( test_log_path , path ) ;
2019-05-23 11:04:37 +03:00
2020-02-06 05:23:44 +03:00
if ( Harness . InCI & & XmlResultParser . IsValidXml ( path , out var xmlType ) ) {
2020-02-04 19:03:38 +03:00
( string resultLine , bool failed , bool crashed ) parseResult = ( null , false , false ) ;
2016-11-15 21:04:37 +03:00
crashed = false ;
2017-06-26 18:34:32 +03:00
try {
2020-02-06 05:23:44 +03:00
var newFilename = XmlResultParser . GetXmlFilePath ( path , xmlType ) ;
2020-02-11 22:03:16 +03:00
// at this point, we have the test results, but we want to be able to have attachments in vsts, so if the format is
// the right one (NUnitV3) add the nodes. ATM only TouchUnit uses V3.
2020-03-10 19:44:18 +03:00
var testRunName = $"{AppName} {Variation}" ;
2020-02-28 04:10:48 +03:00
if ( xmlType = = XmlResultJargon . NUnitV3 ) {
2020-02-19 21:18:34 +03:00
var logFiles = new List < string > ( ) ;
2020-02-14 00:51:14 +03:00
// add our logs AND the logs of the previous task, which is the build task
2020-02-20 02:39:05 +03:00
logFiles . AddRange ( Directory . GetFiles ( Logs . Directory ) ) ;
2020-02-19 19:25:44 +03:00
if ( BuildTask ! = null ) // when using the run command, we do not have a build task, ergo, there are no logs to add.
2020-02-20 02:39:05 +03:00
logFiles . AddRange ( Directory . GetFiles ( BuildTask . LogDirectory ) ) ;
2020-02-11 22:03:16 +03:00
// add the attachments and write in the new filename
2020-02-17 20:27:22 +03:00
// add a final prefix to the file name to make sure that the VSTS test uploaded just pick
// the final version, else we will upload tests more than once
newFilename = XmlResultParser . GetVSTSFilename ( newFilename ) ;
2020-02-19 21:18:34 +03:00
XmlResultParser . UpdateMissingData ( path , newFilename , testRunName , logFiles ) ;
2020-02-11 22:03:16 +03:00
} else {
// rename the path to the correct value
File . Move ( path , newFilename ) ;
}
2020-02-06 05:23:44 +03:00
path = newFilename ;
// write the human readable results in a tmp file, which we later use to step on the logs
var tmpFile = Path . Combine ( Path . GetTempPath ( ) , Guid . NewGuid ( ) . ToString ( ) ) ;
( parseResult . resultLine , parseResult . failed ) = XmlResultParser . GenerateHumanReadableResults ( path , tmpFile , xmlType ) ;
File . Copy ( tmpFile , test_log_path , true ) ;
File . Delete ( tmpFile ) ;
2020-02-04 19:03:38 +03:00
// we do not longer need the tmp file
2020-03-03 21:59:44 +03:00
Logs . AddFile ( path , LogType . XmlLog . ToString ( ) ) ;
2019-05-23 11:04:37 +03:00
return parseResult ;
2020-02-06 05:23:44 +03:00
2017-06-26 18:34:32 +03:00
} catch ( Exception e ) {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Could not parse xml result file: {0}" , e ) ;
2020-02-04 19:03:38 +03:00
// print file for better debugging
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "File data is:" ) ;
MainLog . WriteLine ( new string ( '#' , 10 ) ) ;
2020-02-04 19:03:38 +03:00
using ( var stream = new StreamReader ( path ) ) {
string line ;
while ( ( line = stream . ReadLine ( ) ) ! = null ) {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( line ) ;
2020-02-04 19:03:38 +03:00
}
}
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( new string ( '#' , 10 ) ) ;
MainLog . WriteLine ( "End of xml results." ) ;
2017-06-26 18:34:32 +03:00
if ( timed_out ) {
Harness . LogWrench ( $"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>" ) ;
2019-05-23 11:04:37 +03:00
return parseResult ;
2016-11-15 21:04:37 +03:00
} else {
2017-06-26 18:34:32 +03:00
Harness . LogWrench ( $"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>" ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run crashed" ) ;
2017-06-26 18:34:32 +03:00
crashed = true ;
2019-05-23 11:04:37 +03:00
parseResult . crashed = true ;
return parseResult ;
2016-11-15 21:04:37 +03:00
}
}
2020-02-19 21:18:34 +03:00
} // delete not needed copy
File . Delete ( path ) ;
// not the most efficient way but this just happens when we run
// the tests locally and we usually do not run all tests, we are
// more interested to be efficent on the bots
string resultLine = null ;
using ( var reader = new StreamReader ( test_log_path ) ) {
string line = null ;
bool failed = false ;
while ( ( line = reader . ReadLine ( ) ) ! = null ) {
if ( line . Contains ( "Tests run:" ) ) {
Console . WriteLine ( line ) ;
resultLine = line ;
break ;
} else if ( line . Contains ( "[FAIL]" ) ) {
Console . WriteLine ( line ) ;
failed = true ;
2016-11-15 21:04:37 +03:00
}
2017-06-26 18:34:32 +03:00
}
2020-02-19 21:18:34 +03:00
return ( resultLine , failed , false ) ;
2019-05-23 11:04:37 +03:00
}
}
2017-06-26 18:34:32 +03:00
2020-02-19 21:18:34 +03:00
public bool TestsSucceeded ( string test_log_path , bool timed_out , out bool crashed )
2019-05-23 11:04:37 +03:00
{
2020-02-19 21:18:34 +03:00
var ( resultLine , failed , crashed_out ) = ParseResult ( test_log_path , timed_out , out crashed ) ;
2019-05-23 11:04:37 +03:00
// read the parsed logs in a human readable way
if ( resultLine ! = null ) {
2019-05-24 11:25:00 +03:00
var tests_run = resultLine . Replace ( "Tests run: " , "" ) ;
2017-06-26 18:34:32 +03:00
if ( failed ) {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>" , mode , tests_run ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run failed" ) ;
2016-11-15 21:04:37 +03:00
return false ;
} else {
2017-06-26 18:34:32 +03:00
Harness . LogWrench ( "@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>" , mode , tests_run ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run succeeded" ) ;
2017-06-26 18:34:32 +03:00
return true ;
2016-11-15 21:04:37 +03:00
}
2017-06-26 18:34:32 +03:00
} else if ( timed_out ) {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>" , mode ) ;
return false ;
} else {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>" , mode ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run crashed" ) ;
2017-06-26 18:34:32 +03:00
crashed = true ;
return false ;
2016-11-15 21:04:37 +03:00
}
}
2016-11-04 14:13:20 +03:00
[DllImport ("/usr/lib/libc.dylib")]
extern static IntPtr ttyname ( int filedes ) ;
2016-06-17 18:21:18 +03:00
public async Task < int > RunAsync ( )
2016-05-26 16:06:52 +03:00
{
2016-07-14 22:31:55 +03:00
CrashReportSnapshot crash_reports ;
2020-03-03 21:59:44 +03:00
ILog device_system_log = null ;
ILog listener_log = null ;
2020-03-10 19:44:18 +03:00
ILog run_log = MainLog ;
2016-05-26 16:06:52 +03:00
Initialize ( ) ;
2016-10-11 20:30:11 +03:00
if ( ! isSimulator )
FindDevice ( ) ;
2019-10-14 17:18:46 +03:00
crash_reports = new CrashReportSnapshot ( ) {
2016-10-11 20:30:11 +03:00
Device = ! isSimulator ,
2020-03-10 19:44:18 +03:00
DeviceName = DeviceName ,
2016-10-11 20:30:11 +03:00
Harness = Harness ,
2020-03-10 19:44:18 +03:00
Log = MainLog ,
2016-10-11 20:30:11 +03:00
Logs = Logs ,
LogDirectory = LogDirectory ,
} ;
2016-07-14 22:31:55 +03:00
2019-10-14 17:18:46 +03:00
var args = new List < string > ( ) ;
if ( ! string . IsNullOrEmpty ( Harness . XcodeRoot ) ) {
args . Add ( "--sdkroot" ) ;
args . Add ( Harness . XcodeRoot ) ;
}
2016-05-26 16:06:52 +03:00
for ( int i = - 1 ; i < Harness . Verbosity ; i + + )
2019-10-14 17:18:46 +03:00
args . Add ( "-v" ) ;
args . Add ( "-argument=-connection-mode" ) ;
args . Add ( "-argument=none" ) ; // This will prevent the app from trying to connect to any IDEs
args . Add ( "-argument=-app-arg:-autostart" ) ;
args . Add ( "-setenv=NUNIT_AUTOSTART=true" ) ;
args . Add ( "-argument=-app-arg:-autoexit" ) ;
args . Add ( "-setenv=NUNIT_AUTOEXIT=true" ) ;
args . Add ( "-argument=-app-arg:-enablenetwork" ) ;
args . Add ( "-setenv=NUNIT_ENABLE_NETWORK=true" ) ;
2016-11-15 21:04:37 +03:00
// detect if we are using a jenkins bot.
2020-01-24 02:28:12 +03:00
var useXmlOutput = Harness . InCI ;
2017-07-11 09:58:47 +03:00
if ( useXmlOutput ) {
2019-10-14 17:18:46 +03:00
args . Add ( "-setenv=NUNIT_ENABLE_XML_OUTPUT=true" ) ;
args . Add ( "-setenv=NUNIT_ENABLE_XML_MODE=wrapped" ) ;
2020-02-11 22:03:16 +03:00
args . Add ( "-setenv=NUNIT_XML_VERSION=nunitv3" ) ;
2017-07-11 09:58:47 +03:00
}
2016-11-15 21:04:37 +03:00
2019-07-01 20:02:56 +03:00
if ( Harness . InCI ) {
// We use the 'BUILD_REVISION' variable to detect whether we're running CI or not.
2019-10-14 17:18:46 +03:00
args . Add ( $"-setenv=BUILD_REVISION=${Environment.GetEnvironmentVariable (" BUILD_REVISION ")}" ) ;
2019-07-01 20:02:56 +03:00
}
2019-07-12 17:05:57 +03:00
if ( ! Harness . GetIncludeSystemPermissionTests ( TestPlatform . iOS , ! isSimulator ) )
2019-10-14 17:18:46 +03:00
args . Add ( "-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1" ) ;
2017-08-03 19:58:04 +03:00
2016-05-26 16:06:52 +03:00
if ( isSimulator ) {
2019-10-14 17:18:46 +03:00
args . Add ( "-argument=-app-arg:-hostname:127.0.0.1" ) ;
args . Add ( "-setenv=NUNIT_HOSTNAME=127.0.0.1" ) ;
2016-05-26 16:06:52 +03:00
} else {
var ips = new StringBuilder ( ) ;
var ipAddresses = System . Net . Dns . GetHostEntry ( System . Net . Dns . GetHostName ( ) ) . AddressList ;
for ( int i = 0 ; i < ipAddresses . Length ; i + + ) {
if ( i > 0 )
ips . Append ( ',' ) ;
ips . Append ( ipAddresses [ i ] . ToString ( ) ) ;
}
2020-03-10 19:44:18 +03:00
args . Add ( $"-argument=-app-arg:-hostname:{ips}" ) ;
args . Add ( $"-setenv=NUNIT_HOSTNAME={ips}" ) ;
2016-05-26 16:06:52 +03:00
}
2020-03-03 21:59:44 +03:00
listener_log = Logs . Create ( $"test-{mode}-{Harness.Timestamp}.log" , LogType . TestLog . ToString ( ) , timestamp : ! useXmlOutput ) ;
2020-03-05 14:41:52 +03:00
var transport = ListenerFactory . Create ( mode , listener_log , isSimulator , out var listener , out var fn ) ;
args . Add ( $"-argument=-app-arg:-transport:{transport}" ) ;
args . Add ( $"-setenv=NUNIT_TRANSPORT={transport.ToString ().ToUpper ()}" ) ;
2017-06-26 13:47:37 +03:00
2020-03-05 14:41:52 +03:00
if ( transport = = ListenerTransport . File )
2019-10-14 17:18:46 +03:00
args . Add ( $"-setenv=NUNIT_LOG_FILE={fn}" ) ;
2020-03-05 14:41:52 +03:00
2016-06-18 11:35:57 +03:00
listener . TestLog = listener_log ;
2020-03-10 19:44:18 +03:00
listener . Log = MainLog ;
2016-05-26 16:06:52 +03:00
listener . AutoExit = true ;
listener . Address = System . Net . IPAddress . Any ;
2017-06-26 13:47:37 +03:00
listener . XmlOutput = useXmlOutput ;
2016-05-26 16:06:52 +03:00
listener . Initialize ( ) ;
2019-10-14 17:18:46 +03:00
args . Add ( $"-argument=-app-arg:-hostport:{listener.Port}" ) ;
args . Add ( $"-setenv=NUNIT_HOSTPORT={listener.Port}" ) ;
2016-05-26 16:06:52 +03:00
2017-01-04 21:45:26 +03:00
listener . StartAsync ( ) ;
var cancellation_source = new CancellationTokenSource ( ) ;
var timed_out = false ;
2019-02-11 19:36:13 +03:00
listener . ConnectedTask
2020-03-10 19:44:18 +03:00
. TimeoutAfter ( Harness . LaunchTimeout )
2019-02-11 19:36:13 +03:00
. ContinueWith ( ( v ) = > {
if ( v . IsFaulted ) {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test launch failed: {0}" , v . Exception ) ;
2019-02-11 19:36:13 +03:00
} else if ( v . IsCanceled ) {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test launch was cancelled." ) ;
2019-02-11 19:36:13 +03:00
} else if ( v . Result ) {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run started" ) ;
2019-02-11 19:36:13 +03:00
} else {
cancellation_source . Cancel ( ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test launch timed out after {0} minute(s)." , Harness . LaunchTimeout ) ;
2019-02-11 19:36:13 +03:00
timed_out = true ;
}
} ) . DoNotAwait ( ) ;
2017-01-04 21:45:26 +03:00
2016-08-05 23:28:13 +03:00
foreach ( var kvp in Harness . EnvironmentVariables )
2019-10-14 17:18:46 +03:00
args . Add ( $"-setenv={kvp.Key}={kvp.Value}" ) ;
2016-08-05 23:28:13 +03:00
2016-05-26 16:06:52 +03:00
bool? success = null ;
2016-11-02 17:43:55 +03:00
bool launch_failure = false ;
2016-05-26 16:06:52 +03:00
2020-03-10 19:44:18 +03:00
if ( IsExtension ) {
2017-01-04 21:45:26 +03:00
switch ( extension ) {
case Extension . TodayExtension :
2019-10-14 17:18:46 +03:00
args . Add ( isSimulator ? "--launchsimbundleid" : "--launchdevbundleid" ) ;
args . Add ( "todayviewforextensions:" + BundleIdentifier ) ;
args . Add ( "--observe-extension" ) ;
args . Add ( launchAppPath ) ;
2017-01-04 21:45:26 +03:00
break ;
case Extension . WatchKit2 :
default :
throw new NotImplementedException ( ) ;
}
} else {
2019-10-14 17:18:46 +03:00
args . Add ( isSimulator ? "--launchsim" : "--launchdev" ) ;
args . Add ( launchAppPath ) ;
2017-01-04 21:45:26 +03:00
}
2017-10-10 12:02:24 +03:00
if ( ! isSimulator )
2019-10-14 17:18:46 +03:00
args . Add ( "--disable-memory-limits" ) ;
2017-01-04 21:45:26 +03:00
2019-05-30 18:40:56 +03:00
var timeout = TimeSpan . FromMinutes ( Harness . Timeout * TimeoutMultiplier ) ;
2016-05-26 16:06:52 +03:00
if ( isSimulator ) {
2016-11-16 17:23:11 +03:00
if ( ! await FindSimulatorAsync ( ) )
return 1 ;
2016-06-17 18:21:18 +03:00
2016-11-04 14:13:20 +03:00
if ( mode ! = "watchos" ) {
var stderr_tty = Marshal . PtrToStringAuto ( ttyname ( 2 ) ) ;
if ( ! string . IsNullOrEmpty ( stderr_tty ) ) {
2019-10-14 17:18:46 +03:00
args . Add ( $"--stdout={stderr_tty}" ) ;
args . Add ( $"--stderr={stderr_tty}" ) ;
2016-11-04 14:13:20 +03:00
} else {
2017-11-28 16:27:31 +03:00
var stdout_log = Logs . CreateFile ( $"stdout-{Harness.Timestamp}.log" , "Standard output" ) ;
var stderr_log = Logs . CreateFile ( $"stderr-{Harness.Timestamp}.log" , "Standard error" ) ;
2019-10-14 17:18:46 +03:00
args . Add ( $"--stdout={stdout_log}" ) ;
args . Add ( $"--stderr={stderr_log}" ) ;
2016-11-04 14:13:20 +03:00
}
}
2016-06-16 10:07:47 +03:00
var systemLogs = new List < CaptureLog > ( ) ;
2020-03-10 19:44:18 +03:00
foreach ( var sim in Simulators ) {
2016-06-16 10:07:47 +03:00
// Upload the system log
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "System log for the '{1}' simulator is: {0}" , sim . SystemLog , sim . Name ) ;
2016-06-17 18:21:18 +03:00
bool isCompanion = sim ! = simulator ;
2020-03-04 23:27:52 +03:00
var log = new CaptureLog ( Logs , Path . Combine ( LogDirectory , sim . Name + ".log" ) , sim . SystemLog , entire_file : Harness . Action ! = HarnessAction . Jenkins )
2016-11-17 10:07:09 +03:00
{
2020-03-03 21:59:44 +03:00
Description = isCompanion ? LogType . CompanionSystemLog . ToString ( ) : LogType . SystemLog . ToString ( ) ,
2016-06-17 18:21:18 +03:00
} ;
log . StartCapture ( ) ;
Logs . Add ( log ) ;
systemLogs . Add ( log ) ;
Harness . LogWrench ( "@MonkeyWrench: AddFile: {0}" , log . Path ) ;
2016-06-16 10:07:47 +03:00
}
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "*** Executing {0}/{1} in the simulator ***" , AppName , mode ) ;
2016-06-16 10:07:47 +03:00
2016-06-17 18:21:18 +03:00
if ( EnsureCleanSimulatorState ) {
2020-03-10 19:44:18 +03:00
foreach ( var sim in Simulators )
await sim . PrepareSimulatorAsync ( MainLog , BundleIdentifier ) ;
2016-06-17 18:21:18 +03:00
}
2016-05-26 16:06:52 +03:00
2019-10-14 17:18:46 +03:00
args . Add ( $"--device=:v2:udid={simulator.UDID}" ) ;
2016-05-26 16:06:52 +03:00
2016-06-29 19:21:03 +03:00
await crash_reports . StartCaptureAsync ( ) ;
2016-05-26 16:06:52 +03:00
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Starting test run" ) ;
2016-05-26 16:06:52 +03:00
2020-03-06 17:11:33 +03:00
var result = await ProcessManager . ExecuteCommandAsync ( Harness . MlaunchPath , args , run_log , timeout , cancellation_token : cancellation_source . Token ) ;
2016-06-17 18:21:18 +03:00
if ( result . TimedOut ) {
2016-05-26 16:06:52 +03:00
timed_out = true ;
2016-06-17 18:21:18 +03:00
success = false ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run timed out after {0} minute(s)." , timeout ) ;
2016-06-17 18:21:18 +03:00
} else if ( result . Succeeded ) {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run completed" ) ;
2016-06-17 18:21:18 +03:00
success = true ;
} else {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run failed" ) ;
2016-06-17 18:21:18 +03:00
success = false ;
}
2016-05-26 16:06:52 +03:00
2016-06-17 18:21:18 +03:00
if ( ! success . Value ) {
2016-05-26 16:06:52 +03:00
// find pid
var pid = - 1 ;
2016-06-17 18:21:18 +03:00
using ( var reader = run_log . GetReader ( ) ) {
while ( ! reader . EndOfStream ) {
var line = reader . ReadLine ( ) ;
if ( line . StartsWith ( "Application launched. PID = " , StringComparison . Ordinal ) ) {
var pidstr = line . Substring ( "Application launched. PID = " . Length ) ;
if ( ! int . TryParse ( pidstr , out pid ) )
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Could not parse pid: {0}" , pidstr ) ;
2016-06-17 18:21:18 +03:00
} else if ( line . Contains ( "Xamarin.Hosting: Launched " ) & & line . Contains ( " with pid " ) ) {
var pidstr = line . Substring ( line . LastIndexOf ( ' ' ) ) ;
if ( ! int . TryParse ( pidstr , out pid ) )
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Could not parse pid: {0}" , pidstr ) ;
2016-11-02 17:43:55 +03:00
} else if ( line . Contains ( "error MT1008" ) ) {
launch_failure = true ;
2016-06-17 18:21:18 +03:00
}
2016-05-26 16:06:52 +03:00
}
}
if ( pid > 0 ) {
2016-06-17 18:21:18 +03:00
var launchTimedout = cancellation_source . IsCancellationRequested ;
2016-12-16 14:24:08 +03:00
var timeoutType = launchTimedout ? "Launch" : "Completion" ;
2020-03-10 19:44:18 +03:00
var timeoutValue = launchTimedout ? Harness . LaunchTimeout . TotalSeconds : timeout . TotalSeconds ;
MainLog . WriteLine ( $"{timeoutType} timed out after {timeoutValue} seconds" ) ;
await ProcessManager . KillTreeAsync ( pid , MainLog , true ) ;
2016-05-26 16:06:52 +03:00
} else {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Could not find pid in mtouch output." ) ;
2016-05-26 16:06:52 +03:00
}
}
// cleanup after us
2016-06-17 18:21:18 +03:00
if ( EnsureCleanSimulatorState )
2020-03-10 19:44:18 +03:00
await SimDevice . KillEverythingAsync ( MainLog ) ;
2016-06-16 10:07:47 +03:00
foreach ( var log in systemLogs )
log . StopCapture ( ) ;
2016-05-26 16:06:52 +03:00
} else {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "*** Executing {0}/{1} on device '{2}' ***" , AppName , mode , DeviceName ) ;
2016-05-26 16:06:52 +03:00
2016-08-05 22:03:56 +03:00
if ( mode = = "watchos" ) {
2019-10-14 17:18:46 +03:00
args . Add ( "--attach-native-debugger" ) ; // this prevents the watch from backgrounding the app.
2017-01-04 21:45:26 +03:00
} else {
2019-10-14 17:18:46 +03:00
args . Add ( "--wait-for-exit" ) ;
2016-08-05 22:03:56 +03:00
}
2016-05-26 16:06:52 +03:00
AddDeviceName ( args ) ;
2020-03-10 19:44:18 +03:00
device_system_log = Logs . Create ( $"device-{DeviceName}-{Harness.Timestamp}.log" , "Device log" ) ;
2016-05-26 16:06:52 +03:00
var logdev = new DeviceLogCapturer ( ) {
Harness = Harness ,
2016-06-17 18:21:18 +03:00
Log = device_system_log ,
2020-03-10 19:44:18 +03:00
DeviceName = DeviceName ,
2016-05-26 16:06:52 +03:00
} ;
logdev . StartCapture ( ) ;
2019-11-27 19:44:26 +03:00
try {
await crash_reports . StartCaptureAsync ( ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Starting test run" ) ;
2019-11-27 19:44:26 +03:00
bool waitedForExit = true ;
// We need to check for MT1111 (which means that mlaunch won't wait for the app to exit).
var callbackLog = new CallbackLog ( ( line ) = > {
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as requested because it's not possible to detect app termination when launching using gdbserver
waitedForExit & = line ? . Contains ( "MT1111: " ) ! = true ;
if ( line ? . Contains ( "error MT1007" ) = = true )
launch_failure = true ;
} ) ;
2020-03-10 19:44:18 +03:00
var runLog = Log . CreateAggregatedLog ( callbackLog , MainLog ) ;
2019-11-27 19:44:26 +03:00
var timeoutWatch = Stopwatch . StartNew ( ) ;
2020-03-06 17:11:33 +03:00
var result = await ProcessManager . ExecuteCommandAsync ( Harness . MlaunchPath , args , runLog , timeout , cancellation_token : cancellation_source . Token ) ;
2019-11-27 19:44:26 +03:00
if ( ! waitedForExit & & ! result . TimedOut ) {
// mlaunch couldn't wait for exit for some reason. Let's assume the app exits when the test listener completes.
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Waiting for listener to complete, since mlaunch won't tell." ) ;
2019-11-27 19:44:26 +03:00
if ( ! await listener . CompletionTask . TimeoutAfter ( timeout - timeoutWatch . Elapsed ) ) {
result . TimedOut = true ;
}
2017-11-02 14:46:18 +03:00
}
2019-11-27 19:44:26 +03:00
if ( result . TimedOut ) {
timed_out = true ;
success = false ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run timed out after {0} minute(s)." , timeout . TotalMinutes ) ;
2019-11-27 19:44:26 +03:00
} else if ( result . Succeeded ) {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run completed" ) ;
2019-11-27 19:44:26 +03:00
success = true ;
} else {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run failed" ) ;
2019-11-27 19:44:26 +03:00
success = false ;
}
} finally {
logdev . StopCapture ( ) ;
device_system_log . Dispose ( ) ;
2016-05-26 16:06:52 +03:00
}
// Upload the system log
2016-06-17 18:21:18 +03:00
if ( File . Exists ( device_system_log . FullPath ) ) {
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "A capture of the device log is: {0}" , device_system_log . FullPath ) ;
2016-11-17 10:07:09 +03:00
Harness . LogWrench ( "@MonkeyWrench: AddFile: {0}" , device_system_log . FullPath ) ;
2016-05-26 16:06:52 +03:00
}
}
2018-05-15 16:07:45 +03:00
listener . Cancel ( ) ;
2016-05-26 16:06:52 +03:00
listener . Dispose ( ) ;
// check the final status
var crashed = false ;
2016-06-17 18:21:18 +03:00
if ( File . Exists ( listener_log . FullPath ) ) {
Harness . LogWrench ( "@MonkeyWrench: AddFile: {0}" , listener_log . FullPath ) ;
2020-02-19 21:18:34 +03:00
success = TestsSucceeded ( listener_log . FullPath , timed_out , out crashed ) ;
2016-05-26 16:06:52 +03:00
} else if ( timed_out ) {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>" , mode ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run never launched" ) ;
2016-08-26 16:12:44 +03:00
success = false ;
2016-11-15 21:04:37 +03:00
} else if ( launch_failure ) {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} failed to launch</i></b><br/>" , mode ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run failed to launch" ) ;
2016-11-15 21:04:37 +03:00
success = false ;
2016-05-26 16:06:52 +03:00
} else {
Harness . LogWrench ( "@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>" , mode ) ;
2020-03-10 19:44:18 +03:00
MainLog . WriteLine ( "Test run crashed before it started (no log file produced)" ) ;
2016-05-26 16:06:52 +03:00
crashed = true ;
2016-08-26 16:12:44 +03:00
success = false ;
2016-05-26 16:06:52 +03:00
}
if ( ! success . HasValue )
success = false ;
2016-06-29 19:21:03 +03:00
await crash_reports . EndCaptureAsync ( TimeSpan . FromSeconds ( success . Value ? 0 : 5 ) ) ;
2016-06-17 18:21:18 +03:00
if ( timed_out ) {
2016-06-06 13:48:53 +03:00
Result = TestExecutingResult . TimedOut ;
} else if ( crashed ) {
Result = TestExecutingResult . Crashed ;
2016-06-17 18:21:18 +03:00
} else if ( success . Value ) {
Result = TestExecutingResult . Succeeded ;
2016-06-06 13:48:53 +03:00
} else {
Result = TestExecutingResult . Failed ;
}
2017-02-08 13:44:20 +03:00
// Check crash reports to see if any of them explains why the test run crashed.
if ( ! success . Value ) {
int pid = 0 ;
string crash_reason = null ;
foreach ( var crash in crash_reports . Logs ) {
try {
if ( pid = = 0 ) {
// Find the pid
2020-03-10 19:44:18 +03:00
using ( var log_reader = MainLog . GetReader ( ) ) {
2017-02-08 13:44:20 +03:00
string line ;
while ( ( line = log_reader . ReadLine ( ) ) ! = null ) {
const string str = "was launched with pid '" ;
var idx = line . IndexOf ( str , StringComparison . Ordinal ) ;
if ( idx > 0 ) {
idx + = str . Length ;
var next_idx = line . IndexOf ( '\'' , idx ) ;
if ( next_idx > idx )
int . TryParse ( line . Substring ( idx , next_idx - idx ) , out pid ) ;
}
if ( pid ! = 0 )
break ;
}
}
}
using ( var crash_reader = crash . GetReader ( ) ) {
var text = crash_reader . ReadToEnd ( ) ;
var reader = System . Runtime . Serialization . Json . JsonReaderWriterFactory . CreateJsonReader ( Encoding . UTF8 . GetBytes ( text ) , new XmlDictionaryReaderQuotas ( ) ) ;
var doc = new XmlDocument ( ) ;
doc . Load ( reader ) ;
foreach ( XmlNode node in doc . SelectNodes ( $"/root/processes/item[pid = '" + pid + "']" ) ) {
Console . WriteLine ( node ? . InnerXml ) ;
Console . WriteLine ( node ? . SelectSingleNode ( "reason" ) ? . InnerText ) ;
crash_reason = node ? . SelectSingleNode ( "reason" ) ? . InnerText ;
}
}
2020-02-15 01:27:01 +03:00
if ( crash_reason ! = null ) {
// if in CI, do write an xml error that will be picked as a failure by VSTS
2020-02-17 20:27:22 +03:00
if ( Harness . InCI )
2020-03-10 19:44:18 +03:00
XmlResultParser . GenerateFailure ( Logs , "crash" , AppName , Variation , "AppCrash" , $"App crashed {crash_reason}." , crash_reports . Log . FullPath , Harness . XmlJargon ) ;
2017-02-08 13:44:20 +03:00
break ;
2020-02-15 01:27:01 +03:00
}
2017-02-08 13:44:20 +03:00
} catch ( Exception e ) {
2017-05-31 18:34:26 +03:00
Harness . Log ( 2 , "Failed to process crash report '{1}': {0}" , e . Message , crash . Description ) ;
2017-02-08 13:44:20 +03:00
}
}
if ( ! string . IsNullOrEmpty ( crash_reason ) ) {
if ( crash_reason = = "per-process-limit" ) {
FailureMessage = "Killed due to using too much memory (per-process-limit)." ;
} else {
FailureMessage = $"Killed by the OS ({crash_reason})" ;
}
2020-02-17 20:27:22 +03:00
if ( Harness . InCI )
2020-03-10 19:44:18 +03:00
XmlResultParser . GenerateFailure ( Logs , "crash" , AppName , Variation , "AppCrash" , $"App crashed: {FailureMessage}" , crash_reports . Log . FullPath , Harness . XmlJargon ) ;
2018-03-07 16:48:24 +03:00
} else if ( launch_failure ) {
2020-02-15 01:27:01 +03:00
// same as with a crash
2018-03-07 16:48:24 +03:00
FailureMessage = $"Launch failure" ;
2020-02-17 20:27:22 +03:00
if ( Harness . InCI )
2020-03-10 19:44:18 +03:00
XmlResultParser . GenerateFailure ( Logs , "launch" , AppName , Variation , $"AppLaunch on {DeviceName}" , $"{FailureMessage} on {DeviceName}" , MainLog . FullPath , XmlResultJargon . NUnitV3 ) ;
2020-02-29 16:45:48 +03:00
} else if ( ! isSimulator & & crashed & & string . IsNullOrEmpty ( crash_reason ) & & Harness . InCI ) {
2020-02-18 17:16:10 +03:00
// this happens more that what we would like on devices, the main reason most of the time is that we have had netwoking problems and the
// tcp connection could not be stablished. We are going to report it as an error since we have not parsed the logs, evne when the app might have
2020-02-29 16:45:48 +03:00
// not crashed. We need to check the main_log to see if we do have an tcp issue or not
var isTcp = false ;
2020-03-10 19:44:18 +03:00
using ( var reader = new StreamReader ( MainLog . FullPath ) ) {
2020-02-29 16:45:48 +03:00
string line ;
while ( ( line = reader . ReadLine ( ) ) ! = null ) {
if ( line . Contains ( "Couldn't establish a TCP connection with any of the hostnames" ) ) {
isTcp = true ;
break ;
}
}
}
if ( isTcp )
2020-03-10 19:44:18 +03:00
XmlResultParser . GenerateFailure ( Logs , "tcp-connection" , AppName , Variation , $"TcpConnection on {DeviceName}" , $"Device {DeviceName} could not reach the host over tcp." , MainLog . FullPath , Harness . XmlJargon ) ;
2020-02-18 18:28:06 +03:00
} else if ( timed_out & & Harness . InCI ) {
2020-03-10 19:44:18 +03:00
XmlResultParser . GenerateFailure ( Logs , "timeout" , AppName , Variation , "AppTimeout" , $"Test run timed out after {timeout.TotalMinutes} minute(s)." , MainLog . FullPath , Harness . XmlJargon ) ;
2017-02-08 13:44:20 +03:00
}
}
2016-05-26 16:06:52 +03:00
return success . Value ? 0 : 1 ;
}
2019-10-14 17:18:46 +03:00
public void AddDeviceName ( IList < string > args )
2016-05-26 16:06:52 +03:00
{
2020-03-10 19:44:18 +03:00
AddDeviceName ( args , DeviceName ) ;
2016-05-26 16:06:52 +03:00
}
2019-10-14 17:18:46 +03:00
public static void AddDeviceName ( IList < string > args , string device_name )
2016-05-26 16:06:52 +03:00
{
if ( ! string . IsNullOrEmpty ( device_name ) ) {
2019-10-14 17:18:46 +03:00
args . Add ( "--devname" ) ;
args . Add ( device_name ) ;
2016-05-26 16:06:52 +03:00
}
}
}
2020-02-11 22:03:16 +03:00
}