2019-05-11 13:03:38 +03:00
using System ;
2016-05-26 16:06:52 +03:00
using System.Collections.Generic ;
using System.Diagnostics ;
using System.IO ;
using System.Linq ;
using System.Text ;
using System.Threading ;
2016-06-06 13:48:53 +03:00
using System.Threading.Tasks ;
2016-05-26 16:06:52 +03:00
using System.Xml ;
2020-03-12 00:06:40 +03:00
using Xharness.Hardware ;
2020-03-06 17:11:33 +03:00
using Xharness.Execution ;
2020-03-06 14:00:34 +03:00
using Xharness.Jenkins.TestTasks ;
2020-03-05 14:41:52 +03:00
using Xharness.Listeners ;
2020-03-03 21:59:44 +03:00
using Xharness.Logging ;
2020-03-10 15:12:33 +03:00
using Xharness.Utilities ;
2016-05-26 16:06:52 +03:00
2020-03-17 21:42:21 +03:00
namespace Xharness {
2020-03-16 16:26:46 +03:00
public enum RunMode {
Sim32 ,
Sim64 ,
Classic ,
iOS ,
TvOS ,
WatchOS ,
2016-11-16 17:23:11 +03:00
}
2017-01-04 21:45:26 +03:00
public enum Extension
{
WatchKit2 ,
TodayExtension ,
}
2020-03-16 16:26:46 +03:00
public class AppInformation {
public string AppName { get ; }
public string BundleIdentifier { get ; }
public string AppPath { get ; }
public string LaunchAppPath { get ; }
public Extension ? Extension { get ; }
public AppInformation ( string appName , string bundleIdentifier , string appPath , string launchAppPath , Extension ? extension )
{
AppName = appName ;
BundleIdentifier = bundleIdentifier ;
AppPath = appPath ;
LaunchAppPath = launchAppPath ;
Extension = extension ;
}
}
2020-02-14 00:51:14 +03:00
class AppRunner
2016-05-26 16:06:52 +03:00
{
2020-03-16 16:26:46 +03:00
readonly IProcessManager processManager ;
readonly ISimulatorsLoaderFactory simulatorsLoaderFactory ;
readonly ISimpleListenerFactory listenerFactory ;
readonly IDeviceLoaderFactory devicesLoaderFactory ;
readonly RunMode mode ;
readonly bool isSimulator ;
readonly AppRunnerTarget target ;
readonly string projectFilePath ;
readonly IHarness harness ;
readonly string configuration ;
readonly string variation ;
readonly double timeoutMultiplier ;
readonly BuildToolTask buildTask ;
readonly string logDirectory ;
string deviceName ;
string companionDeviceName ;
ISimulatorDevice [ ] simulators ;
ISimulatorDevice simulator = > simulators [ 0 ] ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
bool ensureCleanSimulatorState = true ;
bool EnsureCleanSimulatorState {
get = > ensureCleanSimulatorState & & string . IsNullOrEmpty ( Environment . GetEnvironmentVariable ( "SKIP_SIMULATOR_SETUP" ) ) ;
set = > ensureCleanSimulatorState = value ;
2020-03-11 12:38:03 +03:00
}
2017-01-04 21:45:26 +03:00
2020-03-16 16:26:46 +03:00
bool IsExtension = > AppInformation . Extension . HasValue ;
public AppInformation AppInformation { get ; }
2017-01-04 21:45:26 +03:00
2020-03-16 16:26:46 +03:00
public TestExecutingResult Result { get ; private set ; }
2017-01-04 21:45:26 +03:00
2020-03-16 16:26:46 +03:00
public string FailureMessage { get ; private set ; }
public ILog MainLog { get ; set ; }
public ILogs Logs { get ; }
public AppRunner ( IProcessManager processManager ,
ISimulatorsLoaderFactory simulatorsFactory ,
ISimpleListenerFactory simpleListenerFactory ,
IDeviceLoaderFactory devicesFactory ,
AppRunnerTarget target ,
IHarness harness ,
ILog mainLog ,
string projectFilePath ,
string configuration ,
string logDirectory = null ,
ISimulatorDevice [ ] simulators = null ,
string deviceName = null ,
string companionDeviceName = null ,
bool ensureCleanSimulatorState = false ,
double timeoutMultiplier = 1 ,
string variation = null ,
BuildToolTask buildTask = null )
{
this . processManager = processManager ? ? throw new ArgumentNullException ( nameof ( processManager ) ) ;
this . simulatorsLoaderFactory = simulatorsFactory ? ? throw new ArgumentNullException ( nameof ( simulatorsFactory ) ) ;
this . listenerFactory = simpleListenerFactory ? ? throw new ArgumentNullException ( nameof ( simpleListenerFactory ) ) ;
this . devicesLoaderFactory = devicesFactory ? ? throw new ArgumentNullException ( nameof ( devicesFactory ) ) ;
this . harness = harness ? ? throw new ArgumentNullException ( nameof ( harness ) ) ;
this . MainLog = mainLog ? ? throw new ArgumentNullException ( nameof ( mainLog ) ) ;
this . projectFilePath = projectFilePath ? ? throw new ArgumentNullException ( nameof ( projectFilePath ) ) ;
this . logDirectory = logDirectory ? ? harness . LogDirectory ;
this . Logs = new Logs ( this . logDirectory ) ;
this . configuration = configuration ;
this . timeoutMultiplier = timeoutMultiplier ;
this . deviceName = deviceName ;
this . companionDeviceName = companionDeviceName ;
this . ensureCleanSimulatorState = ensureCleanSimulatorState ;
this . simulators = simulators ;
this . variation = variation ;
this . buildTask = buildTask ;
this . target = target ;
mode = target . ToRunMode ( ) ;
isSimulator = target . IsSimulator ( ) ;
AppInformation = Initialize ( ) ;
2020-03-11 12:38:03 +03:00
}
2020-02-15 01:28:00 +03:00
2020-03-16 16:26:46 +03:00
AppInformation Initialize ( )
{
var csproj = new XmlDocument ( ) ;
csproj . LoadWithoutNetworkAccess ( projectFilePath ) ;
string appName = csproj . GetAssemblyName ( ) ;
string info_plist_path = csproj . GetInfoPListInclude ( ) ;
2020-03-10 19:44:18 +03:00
2020-03-16 16:26:46 +03:00
var info_plist = new XmlDocument ( ) ;
string plistPath = Path . Combine ( Path . GetDirectoryName ( projectFilePath ) , info_plist_path . Replace ( '\\' , Path . DirectorySeparatorChar ) ) ;
info_plist . LoadWithoutNetworkAccess ( plistPath ) ;
2016-06-06 13:48:53 +03:00
2020-03-16 16:26:46 +03:00
string bundleIdentifier = info_plist . GetCFBundleIdentifier ( ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
Extension ? extension = null ;
string extensionPointIdentifier = info_plist . GetNSExtensionPointIdentifier ( ) ;
if ( ! string . IsNullOrEmpty ( extensionPointIdentifier ) )
extension = extensionPointIdentifier . ParseFromNSExtensionPointIdentifier ( ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
string appPath = Path . Combine ( Path . GetDirectoryName ( projectFilePath ) ,
csproj . GetOutputPath ( isSimulator ? "iPhoneSimulator" : "iPhone" , configuration ) . Replace ( '\\' , Path . DirectorySeparatorChar ) ,
appName + ( extension ! = null ? ".appex" : ".app" ) ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
if ( ! Directory . Exists ( appPath ) )
throw new Exception ( string . Format ( "The app directory {0} does not exist. This is probably a bug in the test harness." , appPath ) ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
string launchAppPath = mode = = RunMode . WatchOS
? Directory . GetDirectories ( Path . Combine ( appPath , "Watch" ) , "*.app" ) [ 0 ]
: appPath ;
2020-03-11 12:38:03 +03:00
2020-03-16 16:26:46 +03:00
return new AppInformation ( appName , bundleIdentifier , appPath , launchAppPath , extension ) ;
2020-03-11 12:38:03 +03:00
}
2016-11-16 17:23:11 +03:00
async Task < bool > FindSimulatorAsync ( )
2016-05-26 16:06:52 +03:00
{
2020-03-11 12:38:03 +03:00
if ( simulators ! = null )
2016-11-16 17:23:11 +03:00
return true ;
2016-06-06 13:48:53 +03:00
2020-03-16 16:26:46 +03:00
var sims = simulatorsLoaderFactory . CreateLoader ( ) ;
2020-03-18 19:10:25 +03:00
await sims . LoadAsync ( Logs . Create ( $"simulator-list-{Helpers.Timestamp}.log" , "Simulator list" ) , false , false ) ;
2020-03-16 16:26:46 +03:00
simulators = await sims . FindAsync ( target , MainLog ) ;
2016-05-26 16:06:52 +03:00
2020-03-11 12:38:03 +03:00
return simulators ! = null ;
2016-05-26 16:06:52 +03:00
}
void FindDevice ( )
{
2020-03-16 16:26:46 +03:00
if ( deviceName ! = null )
2016-05-26 16:06:52 +03:00
return ;
2020-03-16 16:26:46 +03:00
deviceName = Environment . GetEnvironmentVariable ( "DEVICE_NAME" ) ;
if ( ! string . IsNullOrEmpty ( deviceName ) )
2016-05-26 16:06:52 +03:00
return ;
2020-03-16 16:26:46 +03:00
var devs = devicesLoaderFactory . CreateLoader ( ) ;
2016-06-06 13:48:53 +03:00
Task . Run ( async ( ) = >
{
2020-03-16 16:26:46 +03:00
await devs . LoadAsync ( MainLog , false , false ) ;
2016-06-06 13:48:53 +03:00
} ) . Wait ( ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
DeviceClass [ ] deviceClasses ;
2016-06-06 13:48:53 +03:00
switch ( mode ) {
2020-03-16 16:26:46 +03:00
case RunMode . iOS :
deviceClasses = new [ ] { DeviceClass . iPhone , DeviceClass . iPad , DeviceClass . iPod } ;
2016-06-06 13:48:53 +03:00
break ;
2020-03-16 16:26:46 +03:00
case RunMode . WatchOS :
deviceClasses = new [ ] { DeviceClass . Watch } ;
2016-06-06 13:48:53 +03:00
break ;
2020-03-16 16:26:46 +03:00
case RunMode . TvOS :
deviceClasses = new [ ] { DeviceClass . AppleTV } ; // Untested
2016-06-06 13:48:53 +03:00
break ;
default :
2020-03-16 16:26:46 +03:00
throw new ArgumentException ( nameof ( mode ) ) ;
2016-06-06 13:48:53 +03:00
}
2016-05-26 16:06:52 +03:00
2018-02-12 16:43:16 +03:00
var selected = devs . ConnectedDevices . Where ( ( v ) = > deviceClasses . Contains ( v . DeviceClass ) & & v . IsUsableForDebugging ! = false ) ;
2020-03-12 00:06:40 +03:00
IHardwareDevice selected_data ;
2016-06-06 13:48:53 +03:00
if ( selected . Count ( ) = = 0 ) {
2020-03-16 16:26:46 +03:00
throw new NoDeviceFoundException ( $"Could not find any applicable devices with device class(es): {string.Join (" , ", deviceClasses)}" ) ;
2016-06-06 13:48:53 +03:00
} else if ( selected . Count ( ) > 1 ) {
2016-12-22 20:55:25 +03:00
selected_data = selected
. OrderBy ( ( dev ) = >
{
2020-03-11 12:38:03 +03:00
Version v ;
if ( Version . TryParse ( dev . ProductVersion , out v ) )
2016-12-22 20:55:25 +03:00
return v ;
return new Version ( ) ;
} )
. First ( ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Found {0} devices for device class(es) '{1}': '{2}'. Selected: '{3}' (because it has the lowest version)." , selected . Count ( ) , string . Join ( "', '" , deviceClasses ) , string . Join ( "', '" , selected . Select ( ( v ) = > v . Name ) . ToArray ( ) ) , selected_data . Name ) ;
2016-06-06 13:48:53 +03:00
} else {
selected_data = selected . First ( ) ;
}
2016-06-07 19:49:20 +03:00
2020-03-16 16:26:46 +03:00
deviceName = selected_data . Name ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
if ( mode = = RunMode . WatchOS )
companionDeviceName = devs . FindCompanionDevice ( MainLog , selected_data ) . Name ;
2016-05-26 16:06:52 +03:00
}
2019-02-20 08:33:30 +03:00
public async Task < ProcessExecutionResult > InstallAsync ( CancellationToken cancellation_token )
2016-05-26 16:06:52 +03:00
{
if ( isSimulator ) {
// We reset the simulator when running, so a separate install step does not make much sense.
2020-03-16 16:26:46 +03:00
throw new InvalidOperationException ( "Installing to a simulator is not supported." ) ;
2016-05-26 16:06:52 +03:00
}
FindDevice ( ) ;
2019-10-14 17:18:46 +03:00
var args = new List < string > ( ) ;
2020-03-16 16:26:46 +03:00
if ( ! string . IsNullOrEmpty ( harness . XcodeRoot ) ) {
2019-10-14 17:18:46 +03:00
args . Add ( "--sdkroot" ) ;
2020-03-16 16:26:46 +03:00
args . Add ( harness . XcodeRoot ) ;
2019-10-14 17:18:46 +03:00
}
2020-03-16 16:26:46 +03:00
for ( int i = - 1 ; i < harness . Verbosity ; i + + )
2019-10-14 17:18:46 +03:00
args . Add ( "-v" ) ;
2016-05-26 16:06:52 +03:00
2019-10-14 17:18:46 +03:00
args . Add ( "--installdev" ) ;
2020-03-17 21:42:21 +03:00
args . Add ( AppInformation . AppPath ) ;
2020-03-16 16:26:46 +03:00
AddDeviceName ( args , companionDeviceName ? ? deviceName ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
if ( mode = = RunMode . WatchOS ) {
2019-10-14 17:18:46 +03:00
args . Add ( "--device" ) ;
args . Add ( "ios,watchos" ) ;
}
2016-05-26 16:06:52 +03:00
2020-03-17 21:42:21 +03:00
var totalSize = Directory . GetFiles ( AppInformation . AppPath , "*" , SearchOption . AllDirectories ) . Select ( ( v ) = > new FileInfo ( v ) . Length ) . Sum ( ) ;
MainLog . WriteLine ( $"Installing '{AppInformation.AppPath}' to '{companionDeviceName ?? deviceName}'. Size: {totalSize} bytes = {totalSize / 1024.0 / 1024.0:N2} MB" ) ;
2017-10-12 18:08:54 +03:00
2020-03-16 16:26:46 +03:00
return await processManager . ExecuteCommandAsync ( harness . MlaunchPath , args , MainLog , TimeSpan . FromHours ( 1 ) , cancellation_token : cancellation_token ) ;
2016-05-26 16:06:52 +03:00
}
2017-01-04 21:45:26 +03:00
public async Task < ProcessExecutionResult > UninstallAsync ( )
2017-01-02 10:58:46 +03:00
{
if ( isSimulator )
2020-03-16 16:26:46 +03:00
throw new InvalidOperationException ( "Uninstalling from a simulator is not supported." ) ;
2017-01-02 10:58:46 +03:00
FindDevice ( ) ;
2019-10-14 17:18:46 +03:00
var args = new List < string > ( ) ;
2020-03-16 16:26:46 +03:00
if ( ! string . IsNullOrEmpty ( harness . XcodeRoot ) ) {
2019-10-14 17:18:46 +03:00
args . Add ( "--sdkroot" ) ;
2020-03-16 16:26:46 +03:00
args . Add ( harness . XcodeRoot ) ;
2019-10-14 17:18:46 +03:00
}
2020-03-16 16:26:46 +03:00
for ( int i = - 1 ; i < harness . Verbosity ; i + + )
2019-10-14 17:18:46 +03:00
args . Add ( "-v" ) ;
2017-01-02 10:58:46 +03:00
2019-10-14 17:18:46 +03:00
args . Add ( "--uninstalldevbundleid" ) ;
2020-03-17 21:42:21 +03:00
args . Add ( AppInformation . BundleIdentifier ) ;
2020-03-16 16:26:46 +03:00
AddDeviceName ( args , companionDeviceName ? ? deviceName ) ;
2017-01-02 10:58:46 +03:00
2020-03-16 16:26:46 +03:00
return await processManager . ExecuteCommandAsync ( harness . MlaunchPath , args , MainLog , TimeSpan . FromMinutes ( 1 ) ) ;
2016-06-06 13:48:53 +03:00
}
2018-11-16 21:31:40 +03:00
2020-03-16 16:26:46 +03:00
( string resultLine , bool failed , bool crashed ) ParseResult ( AppInformation appInfo , string test_log_path , bool timed_out , out bool crashed )
2016-11-15 21:04:37 +03:00
{
2020-02-19 21:18:34 +03:00
crashed = false ;
if ( ! File . Exists ( test_log_path ) ) {
crashed = true ;
2019-05-23 11:04:37 +03:00
return ( null , false , true ) ; // if we do not have a log file, the test crashes
2020-02-19 21:18:34 +03:00
}
2017-06-26 18:34:32 +03:00
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
2019-05-23 11:04:37 +03:00
//
// On the other hand, the nunit and xunit do not have that data and have to be parsed.
2020-01-28 22:44:34 +03:00
//
// This if statement has a small trick, we found out that internet sharing in some of the bots (VSTS) does not work, in
// that case, we cannot do a TCP connection to xharness to get the log, this is a problem since if we did not get the xml
// from the TCP connection, we are going to fail when trying to read it and not parse it. Therefore, we are not only
// going to check if we are in CI, but also if the listener_log is valid.
2020-02-04 19:03:38 +03:00
var path = Path . ChangeExtension ( test_log_path , "xml" ) ;
2020-02-06 05:23:44 +03:00
XmlResultParser . CleanXml ( test_log_path , path ) ;
2019-05-23 11:04:37 +03:00
2020-03-16 16:26:46 +03:00
if ( harness . InCI & & XmlResultParser . IsValidXml ( path , out var xmlType ) ) {
2020-02-04 19:03:38 +03:00
( string resultLine , bool failed , bool crashed ) parseResult = ( null , false , false ) ;
2016-11-15 21:04:37 +03:00
crashed = false ;
2017-06-26 18:34:32 +03:00
try {
2020-02-06 05:23:44 +03:00
var newFilename = XmlResultParser . GetXmlFilePath ( path , xmlType ) ;
2020-02-11 22:03:16 +03:00
// at this point, we have the test results, but we want to be able to have attachments in vsts, so if the format is
// the right one (NUnitV3) add the nodes. ATM only TouchUnit uses V3.
2020-03-16 16:26:46 +03:00
var testRunName = $"{appInfo.AppName} {variation}" ;
2020-02-28 04:10:48 +03:00
if ( xmlType = = XmlResultJargon . NUnitV3 ) {
2020-02-19 21:18:34 +03:00
var logFiles = new List < string > ( ) ;
2020-02-14 00:51:14 +03:00
// add our logs AND the logs of the previous task, which is the build task
2020-02-20 02:39:05 +03:00
logFiles . AddRange ( Directory . GetFiles ( Logs . Directory ) ) ;
2020-03-16 16:26:46 +03:00
if ( buildTask ! = null ) // when using the run command, we do not have a build task, ergo, there are no logs to add.
logFiles . AddRange ( Directory . GetFiles ( buildTask . LogDirectory ) ) ;
2020-02-11 22:03:16 +03:00
// add the attachments and write in the new filename
2020-02-17 20:27:22 +03:00
// add a final prefix to the file name to make sure that the VSTS test uploaded just pick
// the final version, else we will upload tests more than once
newFilename = XmlResultParser . GetVSTSFilename ( newFilename ) ;
2020-02-19 21:18:34 +03:00
XmlResultParser . UpdateMissingData ( path , newFilename , testRunName , logFiles ) ;
2020-02-11 22:03:16 +03:00
} else {
// rename the path to the correct value
File . Move ( path , newFilename ) ;
}
2020-02-06 05:23:44 +03:00
path = newFilename ;
// write the human readable results in a tmp file, which we later use to step on the logs
var tmpFile = Path . Combine ( Path . GetTempPath ( ) , Guid . NewGuid ( ) . ToString ( ) ) ;
( parseResult . resultLine , parseResult . failed ) = XmlResultParser . GenerateHumanReadableResults ( path , tmpFile , xmlType ) ;
File . Copy ( tmpFile , test_log_path , true ) ;
File . Delete ( tmpFile ) ;
2020-02-04 19:03:38 +03:00
// we do not longer need the tmp file
2020-03-03 21:59:44 +03:00
Logs . AddFile ( path , LogType . XmlLog . ToString ( ) ) ;
2019-05-23 11:04:37 +03:00
return parseResult ;
2020-02-06 05:23:44 +03:00
2017-06-26 18:34:32 +03:00
} catch ( Exception e ) {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Could not parse xml result file: {0}" , e ) ;
2020-02-04 19:03:38 +03:00
// print file for better debugging
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "File data is:" ) ;
MainLog . WriteLine ( new string ( '#' , 10 ) ) ;
2020-02-04 19:03:38 +03:00
using ( var stream = new StreamReader ( path ) ) {
string line ;
while ( ( line = stream . ReadLine ( ) ) ! = null ) {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( line ) ;
2020-02-04 19:03:38 +03:00
}
}
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( new string ( '#' , 10 ) ) ;
MainLog . WriteLine ( "End of xml results." ) ;
2017-06-26 18:34:32 +03:00
if ( timed_out ) {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( $"AddSummary: <b><i>{mode} timed out</i></b><br/>" ) ;
2019-05-23 11:04:37 +03:00
return parseResult ;
2016-11-15 21:04:37 +03:00
} else {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( $"AddSummary: <b><i>{mode} crashed</i></b><br/>" ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run crashed" ) ;
2017-06-26 18:34:32 +03:00
crashed = true ;
2019-05-23 11:04:37 +03:00
parseResult . crashed = true ;
return parseResult ;
2016-11-15 21:04:37 +03:00
}
}
2020-02-19 21:18:34 +03:00
} // delete not needed copy
File . Delete ( path ) ;
// not the most efficient way but this just happens when we run
// the tests locally and we usually do not run all tests, we are
// more interested to be efficent on the bots
string resultLine = null ;
using ( var reader = new StreamReader ( test_log_path ) ) {
string line = null ;
bool failed = false ;
while ( ( line = reader . ReadLine ( ) ) ! = null ) {
if ( line . Contains ( "Tests run:" ) ) {
Console . WriteLine ( line ) ;
resultLine = line ;
break ;
} else if ( line . Contains ( "[FAIL]" ) ) {
Console . WriteLine ( line ) ;
failed = true ;
2016-11-15 21:04:37 +03:00
}
2017-06-26 18:34:32 +03:00
}
2020-02-19 21:18:34 +03:00
return ( resultLine , failed , false ) ;
2019-05-23 11:04:37 +03:00
}
}
2017-06-26 18:34:32 +03:00
2020-03-16 16:26:46 +03:00
public bool TestsSucceeded ( AppInformation appInfo , string test_log_path , bool timed_out , out bool crashed )
2019-05-23 11:04:37 +03:00
{
2020-03-16 16:26:46 +03:00
var ( resultLine , failed , crashed_out ) = ParseResult ( appInfo , test_log_path , timed_out , out crashed ) ;
2019-05-23 11:04:37 +03:00
// read the parsed logs in a human readable way
if ( resultLine ! = null ) {
2019-05-24 11:25:00 +03:00
var tests_run = resultLine . Replace ( "Tests run: " , "" ) ;
2017-06-26 18:34:32 +03:00
if ( failed ) {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddSummary: <b>{0} failed: {1}</b><br/>" , mode , tests_run ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run failed" ) ;
2016-11-15 21:04:37 +03:00
return false ;
} else {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddSummary: {0} succeeded: {1}<br/>" , mode , tests_run ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run succeeded" ) ;
2017-06-26 18:34:32 +03:00
return true ;
2016-11-15 21:04:37 +03:00
}
2017-06-26 18:34:32 +03:00
} else if ( timed_out ) {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddSummary: <b><i>{0} timed out</i></b><br/>" , mode ) ;
2017-06-26 18:34:32 +03:00
return false ;
} else {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddSummary: <b><i>{0} crashed</i></b><br/>" , mode ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run crashed" ) ;
2017-06-26 18:34:32 +03:00
crashed = true ;
return false ;
2016-11-15 21:04:37 +03:00
}
}
2016-06-17 18:21:18 +03:00
public async Task < int > RunAsync ( )
2016-05-26 16:06:52 +03:00
{
2016-07-14 22:31:55 +03:00
CrashReportSnapshot crash_reports ;
2020-03-03 21:59:44 +03:00
ILog device_system_log = null ;
ILog listener_log = null ;
2020-03-16 16:26:46 +03:00
ILog run_log = MainLog ;
2016-05-26 16:06:52 +03:00
2016-10-11 20:30:11 +03:00
if ( ! isSimulator )
FindDevice ( ) ;
2020-03-17 21:42:21 +03:00
crash_reports = new CrashReportSnapshot ( harness , MainLog , Logs , isDevice : ! isSimulator , deviceName ) ;
2016-07-14 22:31:55 +03:00
2019-10-14 17:18:46 +03:00
var args = new List < string > ( ) ;
2020-03-16 16:26:46 +03:00
if ( ! string . IsNullOrEmpty ( harness . XcodeRoot ) ) {
2019-10-14 17:18:46 +03:00
args . Add ( "--sdkroot" ) ;
2020-03-16 16:26:46 +03:00
args . Add ( harness . XcodeRoot ) ;
2019-10-14 17:18:46 +03:00
}
2020-03-16 16:26:46 +03:00
for ( int i = - 1 ; i < harness . Verbosity ; i + + )
2019-10-14 17:18:46 +03:00
args . Add ( "-v" ) ;
args . Add ( "-argument=-connection-mode" ) ;
args . Add ( "-argument=none" ) ; // This will prevent the app from trying to connect to any IDEs
args . Add ( "-argument=-app-arg:-autostart" ) ;
args . Add ( "-setenv=NUNIT_AUTOSTART=true" ) ;
args . Add ( "-argument=-app-arg:-autoexit" ) ;
args . Add ( "-setenv=NUNIT_AUTOEXIT=true" ) ;
args . Add ( "-argument=-app-arg:-enablenetwork" ) ;
args . Add ( "-setenv=NUNIT_ENABLE_NETWORK=true" ) ;
2016-11-15 21:04:37 +03:00
// detect if we are using a jenkins bot.
2020-03-16 16:26:46 +03:00
var useXmlOutput = harness . InCI ;
2017-07-11 09:58:47 +03:00
if ( useXmlOutput ) {
2019-10-14 17:18:46 +03:00
args . Add ( "-setenv=NUNIT_ENABLE_XML_OUTPUT=true" ) ;
args . Add ( "-setenv=NUNIT_ENABLE_XML_MODE=wrapped" ) ;
2020-02-11 22:03:16 +03:00
args . Add ( "-setenv=NUNIT_XML_VERSION=nunitv3" ) ;
2017-07-11 09:58:47 +03:00
}
2016-11-15 21:04:37 +03:00
2020-03-16 16:26:46 +03:00
if ( harness . InCI ) {
2019-07-01 20:02:56 +03:00
// We use the 'BUILD_REVISION' variable to detect whether we're running CI or not.
2019-10-14 17:18:46 +03:00
args . Add ( $"-setenv=BUILD_REVISION=${Environment.GetEnvironmentVariable (" BUILD_REVISION ")}" ) ;
2019-07-01 20:02:56 +03:00
}
2020-03-16 16:26:46 +03:00
if ( ! harness . GetIncludeSystemPermissionTests ( TestPlatform . iOS , ! isSimulator ) )
2019-10-14 17:18:46 +03:00
args . Add ( "-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1" ) ;
2017-08-03 19:58:04 +03:00
2016-05-26 16:06:52 +03:00
if ( isSimulator ) {
2019-10-14 17:18:46 +03:00
args . Add ( "-argument=-app-arg:-hostname:127.0.0.1" ) ;
args . Add ( "-setenv=NUNIT_HOSTNAME=127.0.0.1" ) ;
2016-05-26 16:06:52 +03:00
} else {
var ips = new StringBuilder ( ) ;
var ipAddresses = System . Net . Dns . GetHostEntry ( System . Net . Dns . GetHostName ( ) ) . AddressList ;
for ( int i = 0 ; i < ipAddresses . Length ; i + + ) {
if ( i > 0 )
ips . Append ( ',' ) ;
ips . Append ( ipAddresses [ i ] . ToString ( ) ) ;
}
2020-03-17 12:16:19 +03:00
args . Add ( $"-argument=-app-arg:-hostname:{ips}" ) ;
args . Add ( $"-setenv=NUNIT_HOSTNAME={ips}" ) ;
2016-05-26 16:06:52 +03:00
}
2020-03-18 19:10:25 +03:00
listener_log = Logs . Create ( $"test-{mode.ToString().ToLower()}-{Helpers.Timestamp}.log" , LogType . TestLog . ToString ( ) , timestamp : ! useXmlOutput ) ;
2020-03-17 12:16:19 +03:00
var ( transport , listener , listenerTmpFile ) = listenerFactory . Create ( mode , MainLog , listener_log , isSimulator , true , useXmlOutput ) ;
2020-03-05 14:41:52 +03:00
args . Add ( $"-argument=-app-arg:-transport:{transport}" ) ;
args . Add ( $"-setenv=NUNIT_TRANSPORT={transport.ToString ().ToUpper ()}" ) ;
2017-06-26 13:47:37 +03:00
2020-03-05 14:41:52 +03:00
if ( transport = = ListenerTransport . File )
2020-03-16 16:26:46 +03:00
args . Add ( $"-setenv=NUNIT_LOG_FILE={listenerTmpFile}" ) ;
2020-03-05 14:41:52 +03:00
2016-05-26 16:06:52 +03:00
listener . Initialize ( ) ;
2019-10-14 17:18:46 +03:00
args . Add ( $"-argument=-app-arg:-hostport:{listener.Port}" ) ;
args . Add ( $"-setenv=NUNIT_HOSTPORT={listener.Port}" ) ;
2016-05-26 16:06:52 +03:00
2017-01-04 21:45:26 +03:00
listener . StartAsync ( ) ;
var cancellation_source = new CancellationTokenSource ( ) ;
var timed_out = false ;
2019-02-11 19:36:13 +03:00
listener . ConnectedTask
2020-03-16 16:26:46 +03:00
. TimeoutAfter ( TimeSpan . FromMinutes ( harness . LaunchTimeout ) )
2019-02-11 19:36:13 +03:00
. ContinueWith ( ( v ) = > {
if ( v . IsFaulted ) {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test launch failed: {0}" , v . Exception ) ;
2019-02-11 19:36:13 +03:00
} else if ( v . IsCanceled ) {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test launch was cancelled." ) ;
2019-02-11 19:36:13 +03:00
} else if ( v . Result ) {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run started" ) ;
2019-02-11 19:36:13 +03:00
} else {
cancellation_source . Cancel ( ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test launch timed out after {0} minute(s)." , harness . LaunchTimeout ) ;
2019-02-11 19:36:13 +03:00
timed_out = true ;
}
} ) . DoNotAwait ( ) ;
2017-01-04 21:45:26 +03:00
2020-03-16 16:26:46 +03:00
foreach ( var kvp in harness . EnvironmentVariables )
2019-10-14 17:18:46 +03:00
args . Add ( $"-setenv={kvp.Key}={kvp.Value}" ) ;
2016-08-05 23:28:13 +03:00
2016-05-26 16:06:52 +03:00
bool? success = null ;
2016-11-02 17:43:55 +03:00
bool launch_failure = false ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
if ( IsExtension ) {
2020-03-17 21:42:21 +03:00
switch ( AppInformation . Extension ) {
2017-01-04 21:45:26 +03:00
case Extension . TodayExtension :
2019-10-14 17:18:46 +03:00
args . Add ( isSimulator ? "--launchsimbundleid" : "--launchdevbundleid" ) ;
2020-03-17 21:42:21 +03:00
args . Add ( "todayviewforextensions:" + AppInformation . BundleIdentifier ) ;
2019-10-14 17:18:46 +03:00
args . Add ( "--observe-extension" ) ;
2020-03-17 21:42:21 +03:00
args . Add ( AppInformation . LaunchAppPath ) ;
2017-01-04 21:45:26 +03:00
break ;
case Extension . WatchKit2 :
default :
throw new NotImplementedException ( ) ;
}
} else {
2019-10-14 17:18:46 +03:00
args . Add ( isSimulator ? "--launchsim" : "--launchdev" ) ;
2020-03-17 21:42:21 +03:00
args . Add ( AppInformation . LaunchAppPath ) ;
2017-01-04 21:45:26 +03:00
}
2017-10-10 12:02:24 +03:00
if ( ! isSimulator )
2019-10-14 17:18:46 +03:00
args . Add ( "--disable-memory-limits" ) ;
2017-01-04 21:45:26 +03:00
2020-03-16 16:26:46 +03:00
var timeout = TimeSpan . FromMinutes ( harness . Timeout * timeoutMultiplier ) ;
2016-05-26 16:06:52 +03:00
if ( isSimulator ) {
2016-11-16 17:23:11 +03:00
if ( ! await FindSimulatorAsync ( ) )
return 1 ;
2016-06-17 18:21:18 +03:00
2020-03-16 16:26:46 +03:00
if ( mode ! = RunMode . WatchOS ) {
2020-03-18 19:10:25 +03:00
var stderr_tty = harness . GetStandardErrorTty ( ) ;
2016-11-04 14:13:20 +03:00
if ( ! string . IsNullOrEmpty ( stderr_tty ) ) {
2019-10-14 17:18:46 +03:00
args . Add ( $"--stdout={stderr_tty}" ) ;
args . Add ( $"--stderr={stderr_tty}" ) ;
2016-11-04 14:13:20 +03:00
} else {
2020-03-18 19:10:25 +03:00
var stdout_log = Logs . CreateFile ( $"stdout-{Helpers.Timestamp}.log" , "Standard output" ) ;
var stderr_log = Logs . CreateFile ( $"stderr-{Helpers.Timestamp}.log" , "Standard error" ) ;
2019-10-14 17:18:46 +03:00
args . Add ( $"--stdout={stdout_log}" ) ;
args . Add ( $"--stderr={stderr_log}" ) ;
2016-11-04 14:13:20 +03:00
}
}
2016-06-16 10:07:47 +03:00
var systemLogs = new List < CaptureLog > ( ) ;
2020-03-11 12:38:03 +03:00
foreach ( var sim in simulators ) {
2016-06-16 10:07:47 +03:00
// Upload the system log
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "System log for the '{1}' simulator is: {0}" , sim . SystemLog , sim . Name ) ;
2016-06-17 18:21:18 +03:00
bool isCompanion = sim ! = simulator ;
2020-03-16 16:26:46 +03:00
var log = new CaptureLog ( Logs , Path . Combine ( logDirectory , sim . Name + ".log" ) , sim . SystemLog , entire_file : harness . Action ! = HarnessAction . Jenkins )
2016-11-17 10:07:09 +03:00
{
2020-03-03 21:59:44 +03:00
Description = isCompanion ? LogType . CompanionSystemLog . ToString ( ) : LogType . SystemLog . ToString ( ) ,
2016-06-17 18:21:18 +03:00
} ;
log . StartCapture ( ) ;
Logs . Add ( log ) ;
systemLogs . Add ( log ) ;
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddFile: {0}" , log . Path ) ;
2016-06-16 10:07:47 +03:00
}
2020-03-17 21:42:21 +03:00
MainLog . WriteLine ( "*** Executing {0}/{1} in the simulator ***" , AppInformation . AppName , mode ) ;
2016-06-16 10:07:47 +03:00
2016-06-17 18:21:18 +03:00
if ( EnsureCleanSimulatorState ) {
2020-03-11 12:38:03 +03:00
foreach ( var sim in simulators )
2020-03-17 21:42:21 +03:00
await sim . PrepareSimulatorAsync ( MainLog , AppInformation . BundleIdentifier ) ;
2016-06-17 18:21:18 +03:00
}
2016-05-26 16:06:52 +03:00
2019-10-14 17:18:46 +03:00
args . Add ( $"--device=:v2:udid={simulator.UDID}" ) ;
2016-05-26 16:06:52 +03:00
2016-06-29 19:21:03 +03:00
await crash_reports . StartCaptureAsync ( ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Starting test run" ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
var result = await processManager . ExecuteCommandAsync ( harness . MlaunchPath , args , run_log , timeout , cancellation_token : cancellation_source . Token ) ;
2016-06-17 18:21:18 +03:00
if ( result . TimedOut ) {
2016-05-26 16:06:52 +03:00
timed_out = true ;
2016-06-17 18:21:18 +03:00
success = false ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run timed out after {0} minute(s)." , timeout ) ;
2016-06-17 18:21:18 +03:00
} else if ( result . Succeeded ) {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run completed" ) ;
2016-06-17 18:21:18 +03:00
success = true ;
} else {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run failed" ) ;
2016-06-17 18:21:18 +03:00
success = false ;
}
2016-05-26 16:06:52 +03:00
2016-06-17 18:21:18 +03:00
if ( ! success . Value ) {
2016-05-26 16:06:52 +03:00
// find pid
var pid = - 1 ;
2016-06-17 18:21:18 +03:00
using ( var reader = run_log . GetReader ( ) ) {
while ( ! reader . EndOfStream ) {
var line = reader . ReadLine ( ) ;
if ( line . StartsWith ( "Application launched. PID = " , StringComparison . Ordinal ) ) {
var pidstr = line . Substring ( "Application launched. PID = " . Length ) ;
if ( ! int . TryParse ( pidstr , out pid ) )
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Could not parse pid: {0}" , pidstr ) ;
2016-06-17 18:21:18 +03:00
} else if ( line . Contains ( "Xamarin.Hosting: Launched " ) & & line . Contains ( " with pid " ) ) {
var pidstr = line . Substring ( line . LastIndexOf ( ' ' ) ) ;
if ( ! int . TryParse ( pidstr , out pid ) )
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Could not parse pid: {0}" , pidstr ) ;
2016-11-02 17:43:55 +03:00
} else if ( line . Contains ( "error MT1008" ) ) {
launch_failure = true ;
2016-06-17 18:21:18 +03:00
}
2016-05-26 16:06:52 +03:00
}
}
if ( pid > 0 ) {
2016-06-17 18:21:18 +03:00
var launchTimedout = cancellation_source . IsCancellationRequested ;
2016-12-16 14:24:08 +03:00
var timeoutType = launchTimedout ? "Launch" : "Completion" ;
2020-03-16 16:26:46 +03:00
var timeoutValue = launchTimedout ? harness . LaunchTimeout : timeout . TotalSeconds ;
MainLog . WriteLine ( $"{timeoutType} timed out after {timeoutValue} seconds" ) ;
await processManager . KillTreeAsync ( pid , MainLog , true ) ;
2016-05-26 16:06:52 +03:00
} else {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Could not find pid in mtouch output." ) ;
2016-05-26 16:06:52 +03:00
}
}
// cleanup after us
2016-06-17 18:21:18 +03:00
if ( EnsureCleanSimulatorState )
2020-03-16 16:26:46 +03:00
await SimDevice . KillEverythingAsync ( MainLog ) ;
2016-06-16 10:07:47 +03:00
foreach ( var log in systemLogs )
log . StopCapture ( ) ;
2016-05-26 16:06:52 +03:00
} else {
2020-03-17 21:42:21 +03:00
MainLog . WriteLine ( "*** Executing {0}/{1} on device '{2}' ***" , AppInformation . AppName , mode , deviceName ) ;
2016-05-26 16:06:52 +03:00
2020-03-16 16:26:46 +03:00
if ( mode = = RunMode . WatchOS ) {
2019-10-14 17:18:46 +03:00
args . Add ( "--attach-native-debugger" ) ; // this prevents the watch from backgrounding the app.
2017-01-04 21:45:26 +03:00
} else {
2019-10-14 17:18:46 +03:00
args . Add ( "--wait-for-exit" ) ;
2016-08-05 22:03:56 +03:00
}
2016-05-26 16:06:52 +03:00
AddDeviceName ( args ) ;
2020-03-18 19:10:25 +03:00
device_system_log = Logs . Create ( $"device-{deviceName}-{Helpers.Timestamp}.log" , "Device log" ) ;
2016-05-26 16:06:52 +03:00
var logdev = new DeviceLogCapturer ( ) {
2020-03-16 16:26:46 +03:00
Harness = harness ,
2016-06-17 18:21:18 +03:00
Log = device_system_log ,
2020-03-16 16:26:46 +03:00
DeviceName = deviceName ,
2016-05-26 16:06:52 +03:00
} ;
logdev . StartCapture ( ) ;
2019-11-27 19:44:26 +03:00
try {
await crash_reports . StartCaptureAsync ( ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Starting test run" ) ;
2019-11-27 19:44:26 +03:00
bool waitedForExit = true ;
// We need to check for MT1111 (which means that mlaunch won't wait for the app to exit).
var callbackLog = new CallbackLog ( ( line ) = > {
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as requested because it's not possible to detect app termination when launching using gdbserver
waitedForExit & = line ? . Contains ( "MT1111: " ) ! = true ;
if ( line ? . Contains ( "error MT1007" ) = = true )
launch_failure = true ;
} ) ;
2020-03-16 16:26:46 +03:00
var runLog = Log . CreateAggregatedLog ( callbackLog , MainLog ) ;
2019-11-27 19:44:26 +03:00
var timeoutWatch = Stopwatch . StartNew ( ) ;
2020-03-16 16:26:46 +03:00
var result = await processManager . ExecuteCommandAsync ( harness . MlaunchPath , args , runLog , timeout , cancellation_token : cancellation_source . Token ) ;
2019-11-27 19:44:26 +03:00
if ( ! waitedForExit & & ! result . TimedOut ) {
// mlaunch couldn't wait for exit for some reason. Let's assume the app exits when the test listener completes.
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Waiting for listener to complete, since mlaunch won't tell." ) ;
2019-11-27 19:44:26 +03:00
if ( ! await listener . CompletionTask . TimeoutAfter ( timeout - timeoutWatch . Elapsed ) ) {
result . TimedOut = true ;
}
2017-11-02 14:46:18 +03:00
}
2019-11-27 19:44:26 +03:00
if ( result . TimedOut ) {
timed_out = true ;
success = false ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run timed out after {0} minute(s)." , timeout . TotalMinutes ) ;
2019-11-27 19:44:26 +03:00
} else if ( result . Succeeded ) {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run completed" ) ;
2019-11-27 19:44:26 +03:00
success = true ;
} else {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run failed" ) ;
2019-11-27 19:44:26 +03:00
success = false ;
}
} finally {
logdev . StopCapture ( ) ;
device_system_log . Dispose ( ) ;
2016-05-26 16:06:52 +03:00
}
// Upload the system log
2016-06-17 18:21:18 +03:00
if ( File . Exists ( device_system_log . FullPath ) ) {
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "A capture of the device log is: {0}" , device_system_log . FullPath ) ;
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddFile: {0}" , device_system_log . FullPath ) ;
2016-05-26 16:06:52 +03:00
}
}
2018-05-15 16:07:45 +03:00
listener . Cancel ( ) ;
2016-05-26 16:06:52 +03:00
listener . Dispose ( ) ;
// check the final status
var crashed = false ;
2016-06-17 18:21:18 +03:00
if ( File . Exists ( listener_log . FullPath ) ) {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddFile: {0}" , listener_log . FullPath ) ;
success = TestsSucceeded ( this . AppInformation , listener_log . FullPath , timed_out , out crashed ) ;
2016-05-26 16:06:52 +03:00
} else if ( timed_out ) {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddSummary: <b><i>{0} never launched</i></b><br/>" , mode ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run never launched" ) ;
2016-08-26 16:12:44 +03:00
success = false ;
2016-11-15 21:04:37 +03:00
} else if ( launch_failure ) {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddSummary: <b><i>{0} failed to launch</i></b><br/>" , mode ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run failed to launch" ) ;
2016-11-15 21:04:37 +03:00
success = false ;
2016-05-26 16:06:52 +03:00
} else {
2020-03-17 21:42:21 +03:00
WrenchLog . WriteLine ( "AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>" , mode ) ;
2020-03-16 16:26:46 +03:00
MainLog . WriteLine ( "Test run crashed before it started (no log file produced)" ) ;
2016-05-26 16:06:52 +03:00
crashed = true ;
2016-08-26 16:12:44 +03:00
success = false ;
2016-05-26 16:06:52 +03:00
}
if ( ! success . HasValue )
success = false ;
2016-06-29 19:21:03 +03:00
await crash_reports . EndCaptureAsync ( TimeSpan . FromSeconds ( success . Value ? 0 : 5 ) ) ;
2016-06-17 18:21:18 +03:00
if ( timed_out ) {
2016-06-06 13:48:53 +03:00
Result = TestExecutingResult . TimedOut ;
} else if ( crashed ) {
Result = TestExecutingResult . Crashed ;
2016-06-17 18:21:18 +03:00
} else if ( success . Value ) {
Result = TestExecutingResult . Succeeded ;
2016-06-06 13:48:53 +03:00
} else {
Result = TestExecutingResult . Failed ;
}
2017-02-08 13:44:20 +03:00
// Check crash reports to see if any of them explains why the test run crashed.
if ( ! success . Value ) {
int pid = 0 ;
string crash_reason = null ;
foreach ( var crash in crash_reports . Logs ) {
try {
if ( pid = = 0 ) {
// Find the pid
2020-03-16 16:26:46 +03:00
using ( var log_reader = MainLog . GetReader ( ) ) {
2017-02-08 13:44:20 +03:00
string line ;
while ( ( line = log_reader . ReadLine ( ) ) ! = null ) {
const string str = "was launched with pid '" ;
var idx = line . IndexOf ( str , StringComparison . Ordinal ) ;
if ( idx > 0 ) {
idx + = str . Length ;
var next_idx = line . IndexOf ( '\'' , idx ) ;
if ( next_idx > idx )
int . TryParse ( line . Substring ( idx , next_idx - idx ) , out pid ) ;
}
if ( pid ! = 0 )
break ;
}
}
}
using ( var crash_reader = crash . GetReader ( ) ) {
var text = crash_reader . ReadToEnd ( ) ;
var reader = System . Runtime . Serialization . Json . JsonReaderWriterFactory . CreateJsonReader ( Encoding . UTF8 . GetBytes ( text ) , new XmlDictionaryReaderQuotas ( ) ) ;
var doc = new XmlDocument ( ) ;
doc . Load ( reader ) ;
foreach ( XmlNode node in doc . SelectNodes ( $"/root/processes/item[pid = '" + pid + "']" ) ) {
Console . WriteLine ( node ? . InnerXml ) ;
Console . WriteLine ( node ? . SelectSingleNode ( "reason" ) ? . InnerText ) ;
crash_reason = node ? . SelectSingleNode ( "reason" ) ? . InnerText ;
}
}
2020-02-15 01:27:01 +03:00
if ( crash_reason ! = null ) {
// if in CI, do write an xml error that will be picked as a failure by VSTS
2020-03-16 16:26:46 +03:00
if ( harness . InCI ) {
XmlResultParser . GenerateFailure ( Logs ,
"crash" ,
2020-03-17 21:42:21 +03:00
AppInformation . AppName ,
2020-03-16 16:26:46 +03:00
variation ,
2020-03-17 21:42:21 +03:00
$"App Crash {AppInformation.AppName} {variation}" ,
2020-03-16 16:26:46 +03:00
$"App crashed {crash_reason}." ,
crash_reports . Log . FullPath ,
harness . XmlJargon ) ;
}
2017-02-08 13:44:20 +03:00
break ;
2020-02-15 01:27:01 +03:00
}
2017-02-08 13:44:20 +03:00
} catch ( Exception e ) {
2020-03-16 16:26:46 +03:00
harness . Log ( 2 , "Failed to process crash report '{1}': {0}" , e . Message , crash . Description ) ;
2017-02-08 13:44:20 +03:00
}
}
if ( ! string . IsNullOrEmpty ( crash_reason ) ) {
if ( crash_reason = = "per-process-limit" ) {
FailureMessage = "Killed due to using too much memory (per-process-limit)." ;
} else {
FailureMessage = $"Killed by the OS ({crash_reason})" ;
}
2020-03-16 16:26:46 +03:00
if ( harness . InCI ) {
XmlResultParser . GenerateFailure (
Logs ,
"crash" ,
2020-03-17 21:42:21 +03:00
AppInformation . AppName ,
2020-03-16 16:26:46 +03:00
variation ,
2020-03-17 21:42:21 +03:00
$"App Crash {AppInformation.AppName} {variation}" ,
2020-03-16 16:26:46 +03:00
$"App crashed: {FailureMessage}" ,
crash_reports . Log . FullPath ,
harness . XmlJargon ) ;
}
2018-03-07 16:48:24 +03:00
} else if ( launch_failure ) {
2020-02-15 01:27:01 +03:00
// same as with a crash
2018-03-07 16:48:24 +03:00
FailureMessage = $"Launch failure" ;
2020-03-16 16:26:46 +03:00
if ( harness . InCI ) {
XmlResultParser . GenerateFailure (
Logs ,
"launch" ,
2020-03-17 21:42:21 +03:00
AppInformation . AppName ,
2020-03-16 16:26:46 +03:00
variation ,
2020-03-17 21:42:21 +03:00
$"App Launch {AppInformation.AppName} {variation} on {deviceName}" ,
2020-03-16 16:26:46 +03:00
$"{FailureMessage} on {deviceName}" ,
MainLog . FullPath ,
XmlResultJargon . NUnitV3 ) ;
}
} else if ( ! isSimulator & & crashed & & string . IsNullOrEmpty ( crash_reason ) & & harness . InCI ) {
2020-02-18 17:16:10 +03:00
// this happens more that what we would like on devices, the main reason most of the time is that we have had netwoking problems and the
// tcp connection could not be stablished. We are going to report it as an error since we have not parsed the logs, evne when the app might have
2020-02-29 16:45:48 +03:00
// not crashed. We need to check the main_log to see if we do have an tcp issue or not
var isTcp = false ;
2020-03-16 16:26:46 +03:00
using ( var reader = new StreamReader ( MainLog . FullPath ) ) {
2020-02-29 16:45:48 +03:00
string line ;
while ( ( line = reader . ReadLine ( ) ) ! = null ) {
if ( line . Contains ( "Couldn't establish a TCP connection with any of the hostnames" ) ) {
isTcp = true ;
break ;
}
}
}
2020-03-16 16:26:46 +03:00
if ( isTcp ) {
XmlResultParser . GenerateFailure ( Logs ,
"tcp-connection" ,
2020-03-17 21:42:21 +03:00
AppInformation . AppName ,
2020-03-16 16:26:46 +03:00
variation ,
$"TcpConnection on {deviceName}" ,
$"Device {deviceName} could not reach the host over tcp." ,
MainLog . FullPath ,
harness . XmlJargon ) ;
}
} else if ( timed_out & & harness . InCI ) {
XmlResultParser . GenerateFailure ( Logs ,
"timeout" ,
2020-03-17 21:42:21 +03:00
AppInformation . AppName ,
2020-03-16 16:26:46 +03:00
variation ,
2020-03-17 21:42:21 +03:00
$"App Timeout {AppInformation.AppName} {variation} on bot {deviceName}" ,
$"{AppInformation.AppName} {variation} Test run timed out after {timeout.TotalMinutes} minute(s) on bot {deviceName}." ,
2020-03-16 16:26:46 +03:00
MainLog . FullPath ,
harness . XmlJargon ) ;
2017-02-08 13:44:20 +03:00
}
}
2016-05-26 16:06:52 +03:00
return success . Value ? 0 : 1 ;
}
2019-10-14 17:18:46 +03:00
public void AddDeviceName ( IList < string > args )
2016-05-26 16:06:52 +03:00
{
2020-03-16 16:26:46 +03:00
AddDeviceName ( args , deviceName ) ;
2016-05-26 16:06:52 +03:00
}
2019-10-14 17:18:46 +03:00
public static void AddDeviceName ( IList < string > args , string device_name )
2016-05-26 16:06:52 +03:00
{
if ( ! string . IsNullOrEmpty ( device_name ) ) {
2019-10-14 17:18:46 +03:00
args . Add ( "--devname" ) ;
args . Add ( device_name ) ;
2016-05-26 16:06:52 +03:00
}
}
}
2020-02-11 22:03:16 +03:00
}