[Harness] Add more context to crashes and timeouts. (#8078)

Update the generation of the xml to include a more descriptive title
than "AppCrash". After this commit the name of the testcase and the test
follow this pattern:

- "App Crash {ApplicationName} {Variation}"

This will make it easier to filter since the an example failure will be:

"App Crash Monotouch (Debug)"

Note, the '()' in the example are part of the variation and that is
build by xharness and passed to the runner.

Fixes: https://github.com/xamarin/xamarin-macios/issues/8077
This commit is contained in:
Manuel de la Pena 2020-03-11 17:09:21 -04:00 коммит произвёл GitHub
Родитель 4b3d2d1cb4
Коммит afb11d65cc
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
2 изменённых файлов: 10 добавлений и 10 удалений

Просмотреть файл

@ -856,7 +856,7 @@ namespace Xharness
if (crash_reason != null) {
// if in CI, do write an xml error that will be picked as a failure by VSTS
if (Harness.InCI)
XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, Harness.XmlJargon);
XmlResultParser.GenerateFailure (Logs, "crash", AppName, Variation, $"App Crash {AppName} {Variation}", $"App crashed {crash_reason}.", crash_reports.Log.FullPath, Harness.XmlJargon);
break;
}
} catch (Exception e) {
@ -870,12 +870,12 @@ namespace Xharness
FailureMessage = $"Killed by the OS ({crash_reason})";
}
if (Harness.InCI)
XmlResultParser.GenerateFailure (Logs, "crash", appName, Variation, "AppCrash", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, Harness.XmlJargon);
XmlResultParser.GenerateFailure (Logs, "crash", AppName, Variation, $"App Crash {AppName} {Variation}", $"App crashed: {FailureMessage}", crash_reports.Log.FullPath, Harness.XmlJargon);
} else if (launch_failure) {
// same as with a crash
FailureMessage = $"Launch failure";
if (Harness.InCI)
XmlResultParser.GenerateFailure (Logs, "launch", appName, Variation, $"AppLaunch on {device_name}", $"{FailureMessage} on {device_name}", main_log.FullPath, XmlResultJargon.NUnitV3);
XmlResultParser.GenerateFailure (Logs, "launch", AppName, Variation, $"App Launch {AppName} {Variation} on {DeviceName}", $"{FailureMessage} on {DeviceName}", MainLog.FullPath, XmlResultJargon.NUnitV3);
} else if (!isSimulator && crashed && string.IsNullOrEmpty (crash_reason) && Harness.InCI) {
// this happens more that what we would like on devices, the main reason most of the time is that we have had netwoking problems and the
// tcp connection could not be stablished. We are going to report it as an error since we have not parsed the logs, evne when the app might have
@ -893,7 +893,7 @@ namespace Xharness
if (isTcp)
XmlResultParser.GenerateFailure (Logs, "tcp-connection", appName, Variation, $"TcpConnection on {device_name}", $"Device {device_name} could not reach the host over tcp.", main_log.FullPath, Harness.XmlJargon);
} else if (timed_out && Harness.InCI) {
XmlResultParser.GenerateFailure (Logs, "timeout", appName, Variation, "AppTimeout", $"Test run timed out after {timeout.TotalMinutes} minute(s).", main_log.FullPath, Harness.XmlJargon);
XmlResultParser.GenerateFailure (Logs, "timeout", AppName, Variation, $"App Timeout {AppName} {Variation}", $"Test run timed out after {timeout.TotalMinutes} minute(s).", MainLog.FullPath, Harness.XmlJargon);
}
}

Просмотреть файл

@ -547,11 +547,11 @@ namespace Xharness {
("time", "0"),
("asserts", "1"));
static void WriteNUnitV2TestCase (XmlWriter writer, string message, StreamReader stderr)
static void WriteNUnitV2TestCase (XmlWriter writer, string title, string message, StreamReader stderr)
{
writer.WriteStartElement ("test-case");
WriteAttributes (writer,
("name", "Crash Report"),
("name", title),
("executed", "True"),
("result", "Failure"),
("success", "False"),
@ -593,7 +593,7 @@ namespace Xharness {
writer.WriteAttributeString ("type", "TestFixture");
WriteNUnitV2TestSuiteAttributes (writer, title);
writer.WriteStartElement ("results");
WriteNUnitV2TestCase (writer, message, stderr);
WriteNUnitV2TestCase (writer, title, message, stderr);
writer.WriteEndElement (); // results
writer.WriteEndElement (); // test-suite TextFixture
writer.WriteEndElement (); // results
@ -668,8 +668,8 @@ namespace Xharness {
writer.WriteStartElement ("test-case");
WriteAttributes (writer,
("id", "1"),
("name", "Crash Report"),
("fullname", "Crash Report"),
("name", title),
("fullname", title),
("result", "Failed"),
("time", "0"),
("asserts", "1"));
@ -701,7 +701,7 @@ namespace Xharness {
("passed", "0"),
("failed", "1"),
("skipped", "0"),
("name", "Crash Report"),
("name", title),
("time", "0"));
writer.WriteStartElement ("test");
WriteAttributes (writer,