[mtouch/tests] Add TimingTests (#1007)
* [mtouch/tests] Add TimingTests - New MLaunchTool. - AppLaunchTime (mlaunch): time to launch an application on the simulators. How it works: we first open the simulator by launching a dummy app. This allows us to detect if there are any launch watchdogs. Therefore, for consistency, all measurements are done with the simulator already open. In the case of the AppLaunchTime test, we build the app with the default config and launch it. It's automatically killed by the simulator because it does not have a valid entry point but this is fine because it also kills the process and lets us stop the stopwatch. We then simply log the time performance.
This commit is contained in:
Vincent Dondain
GitHub
Родитель
3eb8aae49a
Коммит
b252093691
|
|
@ -16,6 +16,8 @@ namespace Xamarin.Tests
|
|||
{ "PKG_CONFIG_PATH", "/Applications/Xamarin Studio.app/Contents/MacOS" }
|
||||
};
|
||||
|
||||
const string XS_PATH = "/Applications/Xamarin Studio.app/Contents/Resources";
|
||||
|
||||
static string mt_root;
|
||||
static string ios_destdir;
|
||||
public static string mt_src_root;
|
||||
|
|
@ -232,6 +234,12 @@ namespace Xamarin.Tests
|
|||
}
|
||||
}
|
||||
|
||||
static string XSIphoneDir {
|
||||
get {
|
||||
return Path.Combine (XS_PATH, "lib", "monodevelop", "AddIns", "MonoDevelop.IPhone");
|
||||
}
|
||||
}
|
||||
|
||||
public static string SmcsPath {
|
||||
get {
|
||||
return Path.Combine (SdkBinDir, "smcs");
|
||||
|
|
@ -255,5 +263,11 @@ namespace Xamarin.Tests
|
|||
return Path.Combine (BinDirXI, "mtouch");
|
||||
}
|
||||
}
|
||||
|
||||
public static string MlaunchPath {
|
||||
get {
|
||||
return Path.Combine (XSIphoneDir, "mlaunch.app", "Contents", "MacOS", "mlaunch");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,11 +47,16 @@ namespace Xamarin.Tests
|
|||
}
|
||||
|
||||
public int Execute (string arguments, params string [] args)
|
||||
{
|
||||
return Execute (Configuration.MtouchPath, arguments, args);
|
||||
}
|
||||
|
||||
public int Execute (string toolPath, string arguments, params string [] args)
|
||||
{
|
||||
output.Clear ();
|
||||
output_lines = null;
|
||||
|
||||
var rv = ExecutionHelper.Execute (TestTarget.ToolPath, string.Format (arguments, args), EnvironmentVariables, output, output);
|
||||
var rv = ExecutionHelper.Execute (toolPath, string.Format (arguments, args), EnvironmentVariables, output, output);
|
||||
|
||||
if (rv != 0) {
|
||||
if (output.Length > 0)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
using System;
|
||||
using System.Text;
|
||||
|
||||
using Xamarin.Tests;
|
||||
|
||||
namespace MTouchTests
|
||||
{
|
||||
class MLaunchTool : Tool
|
||||
{
|
||||
public enum MLaunchAction
|
||||
{
|
||||
None,
|
||||
Sim,
|
||||
Dev,
|
||||
}
|
||||
|
||||
public const string None = "None";
|
||||
|
||||
#pragma warning disable 649
|
||||
public int Verbosity;
|
||||
public string SdkRoot;
|
||||
public string Sdk;
|
||||
public string AppPath;
|
||||
#pragma warning restore 649
|
||||
|
||||
// These are a bit smarter
|
||||
public MLaunchAction Action = MLaunchAction.Sim;
|
||||
public MTouch.Profile Profile = MTouch.Profile.Unified;
|
||||
|
||||
string GetVerbosity ()
|
||||
{
|
||||
if (Verbosity == 0)
|
||||
return string.Empty;
|
||||
if (Verbosity > 0)
|
||||
return new string ('-', Verbosity).Replace ("-", "-v ");
|
||||
return new string ('-', -Verbosity).Replace ("-", "-q ");
|
||||
}
|
||||
|
||||
public int Execute ()
|
||||
{
|
||||
return Execute (Configuration.MlaunchPath, BuildArguments ());
|
||||
}
|
||||
|
||||
string BuildArguments ()
|
||||
{
|
||||
var sb = new StringBuilder ();
|
||||
|
||||
switch (Action) {
|
||||
case MLaunchAction.None:
|
||||
break;
|
||||
case MLaunchAction.Sim:
|
||||
if (AppPath == null)
|
||||
throw new Exception ("No AppPath specified.");
|
||||
sb.Append (" --launchsim ").Append (MTouch.Quote (AppPath));
|
||||
break;
|
||||
default:
|
||||
throw new Exception ("MLaunchAction not specified.");
|
||||
}
|
||||
|
||||
if (SdkRoot == None) {
|
||||
// do nothing
|
||||
} else if (!string.IsNullOrEmpty (SdkRoot)) {
|
||||
sb.Append (" --sdkroot ").Append (MTouch.Quote (SdkRoot));
|
||||
} else {
|
||||
sb.Append (" --sdkroot ").Append (MTouch.Quote (Configuration.xcode_root));
|
||||
}
|
||||
|
||||
sb.Append (" ").Append (GetVerbosity ());
|
||||
|
||||
if (Sdk == None) {
|
||||
// do nothing
|
||||
} else if (!string.IsNullOrEmpty (Sdk)) {
|
||||
sb.Append (" --sdk ").Append (Sdk);
|
||||
} else {
|
||||
sb.Append (" --sdk ").Append (MTouch.GetSdkVersion (Profile));
|
||||
}
|
||||
|
||||
string platformName = null;
|
||||
string simType = null;
|
||||
|
||||
switch (Profile) {
|
||||
case MTouch.Profile.Unified:
|
||||
platformName = "iOS";
|
||||
simType = "iPhone-SE";
|
||||
break;
|
||||
case MTouch.Profile.TVOS:
|
||||
platformName = "tvOS";
|
||||
simType = "Apple-TV-1080p";
|
||||
break;
|
||||
default:
|
||||
throw new Exception ("Profile not specified.");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty (platformName) && !string.IsNullOrEmpty (simType)) {
|
||||
var device = string.Format (":v2:runtime=com.apple.CoreSimulator.SimRuntime.{0}-{1},devicetype=com.apple.CoreSimulator.SimDeviceType.{2}", platformName, Configuration.sdk_version.Replace ('.', '-'), simType);
|
||||
sb.Append (" --device:").Append (MTouch.Quote (device));
|
||||
}
|
||||
|
||||
return sb.ToString ();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -627,41 +627,12 @@ namespace MTouchTests
|
|||
[Test]
|
||||
public void ExtensionBuild ()
|
||||
{
|
||||
var testDir = GetTempDirectory ();
|
||||
var app = Path.Combine (testDir, "testApp.app");
|
||||
Directory.CreateDirectory (app);
|
||||
|
||||
try {
|
||||
var exe = CompileTestAppExecutable (testDir);
|
||||
File.WriteAllText (Path.Combine (app, "Info.plist"), @"<?xml version=""1.0"" encoding=""UTF-8""?>
|
||||
<!DOCTYPE plist PUBLIC ""-//Apple//DTD PLIST 1.0//EN"" ""http://www.apple.com/DTDs/PropertyList-1.0.dtd"">
|
||||
<plist version=""1.0"">
|
||||
<dict>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Extensiontest</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>com.xamarin.extensiontest</string>
|
||||
<key>MinimumOSVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>UIDeviceFamily</key>
|
||||
<array>
|
||||
<integer>1</integer>
|
||||
<integer>2</integer>
|
||||
</array>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationPortraitUpsideDown</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
");
|
||||
ExecutionHelper.Execute (TestTarget.ToolPath, string.Format ("-sdkroot " + Configuration.xcode_root + " --sim {0} -sdk {3} --targetver {3} --extension -r:{2} {1}", app, exe, Configuration.XamarinIOSDll, Configuration.sdk_version), hide_output: false);
|
||||
ExecutionHelper.Execute (TestTarget.ToolPath, string.Format ("-sdkroot " + Configuration.xcode_root + " --dev {0} -sdk {3} --targetver {3} --extension -r:{2} {1}", app, exe, Configuration.XamarinIOSDll, Configuration.sdk_version), hide_output: false);
|
||||
} finally {
|
||||
Directory.Delete (testDir, true);
|
||||
using (var mtouch = new MTouchTool ()) {
|
||||
mtouch.CreateTemporaryApp (hasPlist: true);
|
||||
mtouch.Extension = true;
|
||||
mtouch.TargetVer = Configuration.sdk_version;
|
||||
Assert.AreEqual (0, mtouch.Execute (MTouchAction.BuildSim));
|
||||
Assert.AreEqual (0, mtouch.Execute (MTouchAction.BuildDev));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1979,7 +1950,7 @@ class Test {
|
|||
tool.Profile = profile;
|
||||
tool.Verbosity = 5;
|
||||
tool.Cache = Path.Combine (tool.CreateTemporaryDirectory (), "mtouch-test-cache");
|
||||
tool.CreateTemporaryApp ("using UIKit; class C { static void Main (string[] args) { UIApplication.Main (args); } }");
|
||||
tool.CreateTemporaryApp (code: "using UIKit; class C { static void Main (string[] args) { UIApplication.Main (args); } }");
|
||||
tool.FastDev = true;
|
||||
tool.Dlsym = false;
|
||||
|
||||
|
|
@ -2023,12 +1994,12 @@ class Test {
|
|||
return CompileTestAppExecutable (targetDirectory, code, extraArg, profile: MTouch.Profile.Unified);
|
||||
}
|
||||
|
||||
public static string CompileTestAppExecutable (string targetDirectory, string code = null, string extraArg = "", Profile profile = Profile.Unified)
|
||||
public static string CompileTestAppExecutable (string targetDirectory, string code = null, string extraArg = "", Profile profile = Profile.Unified, string appName = "testApp")
|
||||
{
|
||||
if (code == null)
|
||||
code = "public class TestApp { static void Main () { System.Console.WriteLine (typeof (ObjCRuntime.Runtime).ToString ()); } }";
|
||||
|
||||
return CompileTestAppCode ("exe", targetDirectory, code, extraArg, profile);
|
||||
return CompileTestAppCode ("exe", targetDirectory, code, extraArg, profile, appName);
|
||||
}
|
||||
|
||||
public static string CompileTestAppLibrary (string targetDirectory, string code, string extraArg = null, Profile profile = Profile.Unified)
|
||||
|
|
@ -2036,11 +2007,11 @@ class Test {
|
|||
return CompileTestAppCode ("library", targetDirectory, code, extraArg, profile);
|
||||
}
|
||||
|
||||
public static string CompileTestAppCode (string target, string targetDirectory, string code, string extraArg = "", Profile profile = Profile.Unified)
|
||||
public static string CompileTestAppCode (string target, string targetDirectory, string code, string extraArg = "", Profile profile = Profile.Unified, string appName = "testApp")
|
||||
{
|
||||
var ext = target == "exe" ? "exe" : "dll";
|
||||
var cs = Path.Combine (targetDirectory, "testApp.cs");
|
||||
var assembly = Path.Combine (targetDirectory, "testApp." + ext);
|
||||
var assembly = Path.Combine (targetDirectory, appName + "." + ext);
|
||||
var root_library = GetBaseLibrary (profile);
|
||||
|
||||
File.WriteAllText (cs, code);
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ namespace MTouchTests
|
|||
public bool? FastDev;
|
||||
public bool? Dlsym;
|
||||
public string Sdk;
|
||||
public string TargetVer;
|
||||
public string [] References;
|
||||
public string Executable;
|
||||
public string TargetFramework;
|
||||
|
|
@ -165,6 +166,12 @@ namespace MTouchTests
|
|||
sb.Append (" --sdk ").Append (MTouch.GetSdkVersion (Profile));
|
||||
}
|
||||
|
||||
if (TargetVer == None) {
|
||||
// do nothing
|
||||
} else if (!string.IsNullOrEmpty (TargetVer)) {
|
||||
sb.Append (" --targetver ").Append (TargetVer);
|
||||
}
|
||||
|
||||
if (Debug.HasValue && Debug.Value)
|
||||
sb.Append (" --debug");
|
||||
|
||||
|
|
@ -326,14 +333,79 @@ namespace MTouchTests
|
|||
}
|
||||
}
|
||||
|
||||
public void CreateTemporaryApp (string code = null)
|
||||
string CreatePlist (MTouch.Profile profile, string appName)
|
||||
{
|
||||
string plist = null;
|
||||
|
||||
switch (profile) {
|
||||
case MTouch.Profile.Unified:
|
||||
plist = string.Format (@"<?xml version=""1.0"" encoding=""UTF-8""?>
|
||||
<!DOCTYPE plist PUBLIC ""-//Apple//DTD PLIST 1.0//EN"" ""http://www.apple.com/DTDs/PropertyList-1.0.dtd"">
|
||||
<plist version=""1.0"">
|
||||
<dict>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>{0}</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>com.xamarin.{0}</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>{0}</string>
|
||||
<key>MinimumOSVersion</key>
|
||||
<string>{1}</string>
|
||||
<key>UIDeviceFamily</key>
|
||||
<array>
|
||||
<integer>1</integer>
|
||||
<integer>2</integer>
|
||||
</array>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationPortraitUpsideDown</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
", appName, MTouch.GetSdkVersion (Profile));
|
||||
break;
|
||||
case MTouch.Profile.TVOS:
|
||||
plist = string.Format (@"<?xml version=""1.0"" encoding=""UTF-8""?>
|
||||
<!DOCTYPE plist PUBLIC ""-//Apple//DTD PLIST 1.0//EN"" ""http://www.apple.com/DTDs/PropertyList-1.0.dtd"">
|
||||
<plist version=""1.0"">
|
||||
<dict>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Extensiontest</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>com.xamarin.{0}</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>{0}</string>
|
||||
<key>MinimumOSVersion</key>
|
||||
<string>{1}</string>
|
||||
<key>UIDeviceFamily</key>
|
||||
<array>
|
||||
<integer>3</integer>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
", appName, MTouch.GetSdkVersion (Profile));
|
||||
break;
|
||||
default:
|
||||
throw new Exception ("Profile not specified.");
|
||||
}
|
||||
|
||||
return plist;
|
||||
}
|
||||
|
||||
public void CreateTemporaryApp (MTouch.Profile profile = MTouch.Profile.Unified, bool hasPlist = false, string appName = "testApp", string code = null)
|
||||
{
|
||||
var testDir = CreateTemporaryDirectory ();
|
||||
var app = Path.Combine (testDir, "testApp.app");
|
||||
var app = Path.Combine (testDir, appName + ".app");
|
||||
Directory.CreateDirectory (app);
|
||||
|
||||
AppPath = app;
|
||||
Executable = MTouch.CompileTestAppExecutable (testDir, code: code, profile: Profile);
|
||||
Executable = MTouch.CompileTestAppExecutable (testDir, code, "", Profile, appName);
|
||||
|
||||
if (hasPlist)
|
||||
File.WriteAllText (Path.Combine (app, "Info.plist"), CreatePlist (profile, appName));
|
||||
}
|
||||
|
||||
public void CreateTemporaryWatchKitExtension (string code = null)
|
||||
|
|
@ -407,7 +479,7 @@ public partial class NotificationController : WKUserNotificationInterfaceControl
|
|||
{
|
||||
if (AppPath != null)
|
||||
throw new Exception ("There already is an App directory");
|
||||
|
||||
|
||||
AppPath = Path.Combine (CreateTemporaryDirectory (), "testApp.app");
|
||||
Directory.CreateDirectory (AppPath);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
using System;
|
||||
using System.Diagnostics;
|
||||
using MTouchTests;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace Xamarin.Profiler
|
||||
{
|
||||
[TestFixture (MTouch.Profile.Unified)]
|
||||
[TestFixture (MTouch.Profile.TVOS)]
|
||||
public class TimingTests
|
||||
{
|
||||
MTouch.Profile profile;
|
||||
|
||||
public TimingTests (MTouch.Profile profile)
|
||||
{
|
||||
this.profile = profile;
|
||||
|
||||
// Create dummy app to initialize the simulator.
|
||||
using (var buildTool = new MTouchTool ()) {
|
||||
buildTool.Profile = profile;
|
||||
buildTool.CreateTemporaryApp (profile, true);
|
||||
buildTool.Execute (MTouchAction.BuildSim);
|
||||
var mlaunch = new MLaunchTool ();
|
||||
mlaunch.AppPath = buildTool.AppPath;
|
||||
mlaunch.Profile = profile;
|
||||
mlaunch.Execute ();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Time to launch an application on the simulators.</summary>
|
||||
/// <remarks>
|
||||
/// Note: the measurement is being done with the simulator already open and the app not yet installed.</remarks>
|
||||
[Test]
|
||||
public void AppLaunchTime ()
|
||||
{
|
||||
using (var buildTool = new MTouchTool ()) {
|
||||
buildTool.Profile = profile;
|
||||
buildTool.CreateTemporaryApp (profile, true, "AppLaunchTime" + profile);
|
||||
|
||||
buildTool.Execute (MTouchAction.BuildSim);
|
||||
|
||||
var sw = new Stopwatch ();
|
||||
var launchTool = new MLaunchTool ();
|
||||
launchTool.AppPath = buildTool.AppPath;
|
||||
launchTool.Profile = profile;
|
||||
|
||||
sw.Start ();
|
||||
launchTool.Execute ();
|
||||
sw.Stop ();
|
||||
|
||||
Console.WriteLine ("TimingTests - AppLaunchTime ({0}): {1} seconds", profile, sw.Elapsed.TotalSeconds.ToString ("#.000"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -45,6 +45,8 @@
|
|||
</Compile>
|
||||
<Compile Include="SdkTest.cs" />
|
||||
<Compile Include="MTouchTool.cs" />
|
||||
<Compile Include="TimingTests.cs" />
|
||||
<Compile Include="MLaunchTool.cs" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
|
||||
<ItemGroup>
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче